From 49b05125980e486bec7262e33afc6a14befa4626 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Tue, 17 Sep 2024 09:01:04 -0400 Subject: [PATCH 01/25] Update docker-compose, .env.example files. add axios dep --- .env.example | 54 +++++++----- .gitignore | 3 +- Makefile | 6 +- dac-api-compose/docker-compose.yaml | 8 +- package-lock.json | 126 ++++++++++++++++------------ package.json | 2 + 6 files changed, 115 insertions(+), 84 deletions(-) diff --git a/.env.example b/.env.example index 0bd1c08..7fea213 100644 --- a/.env.example +++ b/.env.example @@ -26,7 +26,7 @@ JWT_TOKEN_PUBLIC_KEY= ############ # true or false VAULT_ENABLED=false -VAULT_SECRETS_PATH=/service/secrets_v1 +VAULT_SECRETS_PATH= VAULT_URL= VAULT_ROLE= # for local development/testing @@ -48,34 +48,34 @@ DACO_REVIEW_POLICY_NAME=DACO-REVIEW ############ # Storage # ############ -OBJECT_STORAGE_ENDPOINT=https://object.cancercollaboratory.org:9080 -OBJECT_STORAGE_REGION= -OBJECT_STORAGE_BUCKET= -OBJECT_STORAGE_KEY= -OBJECT_STORAGE_SECRET= +OBJECT_STORAGE_ENDPOINT=http://localhost:8085 +OBJECT_STORAGE_REGION=nova +OBJECT_STORAGE_BUCKET=daco +OBJECT_STORAGE_KEY=minio +OBJECT_STORAGE_SECRET=minio123 OBJECT_STORAGE_TIMEOUT_MILLIS=5000 ############ # EMAIL # ############ -EMAIL_HOST=smtp.gmail.com -EMAIL_PORT=587 +EMAIL_HOST=localhost +EMAIL_PORT=1025 EMAIL_USER= EMAIL_PASSWORD= -EMAIL_FROM_ADDRESS= -EMAIL_FROM_NAME= -EMAIL_DACO_ADDRESS= +EMAIL_FROM_ADDRESS=daco@example.com +EMAIL_FROM_NAME=DacoAdmin +EMAIL_DACO_ADDRESS=daco@example.com # for emails directed to daco reviewers -EMAIL_REVIEWER_FIRSTNAME= -EMAIL_REVIEWER_LASTNAME= +EMAIL_REVIEWER_FIRSTNAME=DACO +EMAIL_REVIEWER_LASTNAME=ADMIN DCC_MAILING_LIST= DACO_SURVEY_URL= ############## # UI # ############## -DACO_UI_BASE_URL=https://dac.dev.argo.cancercollaboratory.org +DACO_UI_BASE_URL=http://localhost:3000 DACO_UI_APPLICATION_SECTION_PATH=/applications/{id}?section={section} ############## @@ -88,16 +88,16 @@ FILE_UPLOAD_LIMIT=#in bytes x * 1024 * 1024 ############## # ATTESTATION -ATTESTATION_UNIT_COUNT= -ATTESTATION_UNIT_OF_TIME= -DAYS_TO_ATTESTATION= +ATTESTATION_UNIT_COUNT=1 +ATTESTATION_UNIT_OF_TIME=years +DAYS_TO_ATTESTATION=45 # EXPIRY -DAYS_TO_EXPIRY_1= -DAYS_TO_EXPIRY_2= -DAYS_POST_EXPIRY= -EXPIRY_UNIT_COUNT= -EXPIRY_UNIT_OF_TIME= +DAYS_TO_EXPIRY_1=90 +DAYS_TO_EXPIRY_2=45 +DAYS_POST_EXPIRY=90 +EXPIRY_UNIT_COUNT=2 +EXPIRY_UNIT_OF_TIME=years ############# # Daco Encryption @@ -109,3 +109,13 @@ DACO_ENCRYPTION_KEY= ############# FEATURE_RENEWAL_ENABLED=false FEATURE_ADMIN_PAUSE_ENABLED=false + +############# +# EGA +############# +EGA_CLIENT_ID= +EGA_AUTH_HOST= +EGA_AUTH_REALM_NAME= +EGA_API_URL= +EGA_USERNAME= +EGA_PASSWORD= \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1a9a185..c074087 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,5 @@ typings/ dist/ # misc -.DS_Store \ No newline at end of file +.DS_Store +.vscode/settings.json \ No newline at end of file diff --git a/Makefile b/Makefile index da596ff..f4363bc 100644 --- a/Makefile +++ b/Makefile @@ -4,15 +4,15 @@ debug: dcompose #run the docker compose file dcompose: - docker-compose -f dac-api-compose/docker-compose.yaml up -d + docker compose -f dac-api-compose/docker-compose.yaml up -d # run all tests verify: npm run test stop: - docker-compose -f dac-api-compose/docker-compose.yaml down --remove-orphans + docker compose -f dac-api-compose/docker-compose.yaml down --remove-orphans # delete. everything. nuke: - docker-compose -f dac-api-compose/docker-compose.yaml down --volumes --remove-orphans + docker compose -f dac-api-compose/docker-compose.yaml down --volumes --remove-orphans diff --git a/dac-api-compose/docker-compose.yaml b/dac-api-compose/docker-compose.yaml index a8486c0..59ed8c4 100644 --- a/dac-api-compose/docker-compose.yaml +++ b/dac-api-compose/docker-compose.yaml @@ -1,8 +1,6 @@ -version: '3.8' - services: vault: - image: vault + image: vault:1.13.3 volumes: - $PWD/logs/:/tmp/logs - ./vault:/scripts @@ -47,8 +45,8 @@ services: # for email services mailhog: - image: mailhog/mailhog - container_name: 'mailhog' + image: jcalonso/mailhog:latest + container_name: mailhog ports: - '1025:1025' - '8025:8025' diff --git a/package-lock.json b/package-lock.json index d20ce8a..87eeb93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "dac-api", - "version": "1.7.0", + "version": "1.8.9", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "dac-api", - "version": "1.7.0", + "version": "1.8.9", "license": "AGPL-3.0", "dependencies": { "@aws-sdk/client-s3": "^3.18.0", @@ -14,6 +14,7 @@ "@overture-stack/ego-token-middleware": "^2.4.1", "archiver": "^5.3.0", "aws-sdk": "^2.923.0", + "axios": "^1.7.7", "body-parser": "^1.19.0", "cd": "^0.3.3", "connect-mongo": "^4.4.1", @@ -65,6 +66,7 @@ "@types/errorhandler": "^1.5.0", "@types/express": "^4.17.11", "@types/express-fileupload": "^1.1.6", + "@types/jsonwebtoken": "^9.0.7", "@types/lodash": "^4.14.168", "@types/memoizee": "^0.4.5", "@types/migrate-mongo": "^8.1.0", @@ -1804,29 +1806,6 @@ "zod": "^3.19.1" } }, - "node_modules/@overture-stack/ego-token-middleware/node_modules/axios": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.2.0.tgz", - "integrity": "sha512-zT7wZyNYu3N5Bu0wuZ6QccIf93Qk1eV8LOewxgjOZFd2DenOs98cJ7+Y6703d0wkaXGY6/nZd4EweJaHz9uzQw==", - "dependencies": { - "follow-redirects": "^1.15.0", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, - "node_modules/@overture-stack/ego-token-middleware/node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/@sindresorhus/is": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", @@ -2059,6 +2038,15 @@ "@types/node": "*" } }, + "node_modules/@types/jsonwebtoken": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.7.tgz", + "integrity": "sha512-ugo316mmTYBl2g81zDFnZ7cfxlut3o+/EQdaP7J8QN2kY6lJ22hmQYCK5EHcJHbrW+dkCGSCPgbG8JtYj6qSrg==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/lodash": { "version": "4.14.168", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.168.tgz", @@ -2765,6 +2753,29 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" }, + "node_modules/axios": { + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/axios/node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -4487,9 +4498,9 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "node_modules/follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", "funding": [ { "type": "individual", @@ -11052,28 +11063,6 @@ "path-to-regexp": "^6.2.0", "url-join": "^4.0.1", "zod": "^3.19.1" - }, - "dependencies": { - "axios": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.2.0.tgz", - "integrity": "sha512-zT7wZyNYu3N5Bu0wuZ6QccIf93Qk1eV8LOewxgjOZFd2DenOs98cJ7+Y6703d0wkaXGY6/nZd4EweJaHz9uzQw==", - "requires": { - "follow-redirects": "^1.15.0", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, - "form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - } - } } }, "@sindresorhus/is": { @@ -11299,6 +11288,15 @@ "@types/node": "*" } }, + "@types/jsonwebtoken": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.7.tgz", + "integrity": "sha512-ugo316mmTYBl2g81zDFnZ7cfxlut3o+/EQdaP7J8QN2kY6lJ22hmQYCK5EHcJHbrW+dkCGSCPgbG8JtYj6qSrg==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/lodash": { "version": "4.14.168", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.168.tgz", @@ -11926,6 +11924,28 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" }, + "axios": { + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", + "requires": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + }, + "dependencies": { + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + } + } + }, "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -13324,9 +13344,9 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==" }, "forever-agent": { "version": "0.6.1", diff --git a/package.json b/package.json index 1028769..92c70df 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "@types/errorhandler": "^1.5.0", "@types/express": "^4.17.11", "@types/express-fileupload": "^1.1.6", + "@types/jsonwebtoken": "^9.0.7", "@types/lodash": "^4.14.168", "@types/memoizee": "^0.4.5", "@types/migrate-mongo": "^8.1.0", @@ -84,6 +85,7 @@ "@overture-stack/ego-token-middleware": "^2.4.1", "archiver": "^5.3.0", "aws-sdk": "^2.923.0", + "axios": "^1.7.7", "body-parser": "^1.19.0", "cd": "^0.3.3", "connect-mongo": "^4.4.1", From 4227b2bedcc6feec377a72de2b77390960512eb2 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Tue, 17 Sep 2024 09:08:08 -0400 Subject: [PATCH 02/25] Update config, secrets values. Create egaClient with token fetch --- src/config.ts | 12 ++++++ src/jobs/ega/egaClient.ts | 86 ++++++++++++++++++++++++++++++++++++++ src/routes/applications.ts | 22 +--------- src/routes/utils.ts | 41 ++++++++++++++++++ src/secrets.ts | 8 ++++ src/utils/constants.ts | 5 +++ 6 files changed, 153 insertions(+), 21 deletions(-) create mode 100644 src/jobs/ega/egaClient.ts create mode 100644 src/routes/utils.ts diff --git a/src/config.ts b/src/config.ts index 8a0694f..4138772 100644 --- a/src/config.ts +++ b/src/config.ts @@ -90,6 +90,12 @@ export interface AppConfig { renewalEnabled: boolean; adminPauseEnabled: boolean; }; + ega: { + clientId: string; + authHost: string; + authRealmName: string; + apiUrl: string; + }; } // Mongo @@ -200,6 +206,12 @@ const buildAppContext = (): AppConfig => { renewalEnabled: process.env.FEATURE_RENEWAL_ENABLED === 'true', adminPauseEnabled: process.env.FEATURE_ADMIN_PAUSE_ENABLED === 'true', }, + ega: { + clientId: checkIsDefined(process.env.EGA_CLIENT_ID), + authHost: checkIsDefined(process.env.EGA_AUTH_HOST), + authRealmName: checkIsDefined(process.env.EGA_AUTH_REALM_NAME), + apiUrl: checkIsDefined(process.env.EGA_API_URL), + }, }; return config; }; diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts new file mode 100644 index 0000000..3d10994 --- /dev/null +++ b/src/jobs/ega/egaClient.ts @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import axios from 'axios'; +import urlJoin from 'url-join'; +import { getAppConfig } from '../../config'; +import getAppSecrets from '../../secrets'; +import { EGA_GRANT_TYPE, EGA_REALMS_PATH, EGA_TOKEN_ENDPOINT } from '../../utils/constants'; + +// initialize idp client +const initIdpClient = () => { + const { + ega: { authHost }, + } = getAppConfig(); + return axios.create({ + baseURL: authHost, + }); +}; +const idpClient = initIdpClient(); + +// initialize API client +const initApiAxiosClient = () => { + const { + ega: { apiUrl }, + } = getAppConfig(); + return axios.create({ + baseURL: apiUrl, + }); +}; +const apiAxiosClient = initApiAxiosClient(); + +/** + * POST request to retrieve an accessToken for the EGA API client + * @returns Promise + */ +const getAccessToken = async (): Promise => { + const { + ega: { authRealmName, clientId }, + } = getAppConfig(); + const { + auth: { egaUsername, egaPassword }, + } = await getAppSecrets(); + + const response = await idpClient.post( + urlJoin(EGA_REALMS_PATH, authRealmName, EGA_TOKEN_ENDPOINT), + { + grant_type: EGA_GRANT_TYPE, + + client_id: clientId, + username: egaUsername, + password: egaPassword, + }, + { + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + }, + ); + + const token = response.data; + return token; +}; + +/** + * Fetches access token and attaches to Axios instance headers for apiClient + * @returns API functions that use authenticated Axios instance + */ +export const egaApiClient = async () => { + const token = await getAccessToken(); +}; diff --git a/src/routes/applications.ts b/src/routes/applications.ts index b6f63dd..0d7c441 100644 --- a/src/routes/applications.ts +++ b/src/routes/applications.ts @@ -41,6 +41,7 @@ import { Storage } from '../storage'; import runAllJobs from '../jobs/runAllJobs'; import { sendEncryptedApprovedUsersEmail } from '../jobs/approvedUsersEmail'; import { isUserJwt } from '../utils/permissions'; +import { validateId, validateType } from './utils'; const createApplicationsRouter = ( config: AppConfig, @@ -442,25 +443,4 @@ const createApplicationsRouter = ( return router; }; -function validateId(id: string) { - if (!id) { - throw new BadRequest('id is required'); - } - if (!id.startsWith('DACO-')) { - throw new BadRequest('Invalid id'); - } - return id; -} - -function validateType(type: string) { - if ( - !['ETHICS', 'SIGNED_APP', 'APPROVED_PDF', 'ethics', 'signed_app', 'approved_pdf'].includes(type) - ) { - throw new BadRequest( - 'unknown document type, should be one of ETHICS, SIGNED_APP or APPROVED_PDF', - ); - } - return type.toUpperCase(); -} - export default createApplicationsRouter; diff --git a/src/routes/utils.ts b/src/routes/utils.ts new file mode 100644 index 0000000..5bcb12e --- /dev/null +++ b/src/routes/utils.ts @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { BadRequest } from '../utils/errors'; + +export function validateId(id: string) { + if (!id) { + throw new BadRequest('id is required'); + } + if (!id.startsWith('DACO-')) { + throw new BadRequest('Invalid id'); + } + return id; +} + +export function validateType(type: string) { + if ( + !['ETHICS', 'SIGNED_APP', 'APPROVED_PDF', 'ethics', 'signed_app', 'approved_pdf'].includes(type) + ) { + throw new BadRequest( + 'unknown document type, should be one of ETHICS, SIGNED_APP or APPROVED_PDF', + ); + } + return type.toUpperCase(); +} diff --git a/src/secrets.ts b/src/secrets.ts index 83fe007..173a022 100644 --- a/src/secrets.ts +++ b/src/secrets.ts @@ -2,6 +2,7 @@ import * as dotenv from 'dotenv'; import logger from './logger'; import * as vault from './vault'; +import { fetchPublicKeyFromRealm } from './jobs/ega/publicKey'; export interface MongoSecrets { dbUser: string; @@ -19,6 +20,9 @@ export interface AppSecrets { }; auth: { dacoEncryptionKey: string; + egaUsername: string; + egaPassword: string; + egaPublicKey: string; }; storage: { key: string; @@ -48,6 +52,7 @@ const loadVaultSecrets = async () => { const buildSecrets = async (vaultSecrets: Record = {}): Promise => { logger.info('Building app secrets...'); + const publicKey = await fetchPublicKeyFromRealm(); secrets = { email: { auth: { @@ -57,6 +62,9 @@ const buildSecrets = async (vaultSecrets: Record = {}): Promise Date: Thu, 19 Sep 2024 13:37:33 -0400 Subject: [PATCH 03/25] initial refresh token logic --- .env.example | 2 +- .gitignore | 2 +- src/jobs/ega/egaClient.ts | 71 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 7fea213..70554ed 100644 --- a/.env.example +++ b/.env.example @@ -118,4 +118,4 @@ EGA_AUTH_HOST= EGA_AUTH_REALM_NAME= EGA_API_URL= EGA_USERNAME= -EGA_PASSWORD= \ No newline at end of file +EGA_PASSWORD= diff --git a/.gitignore b/.gitignore index c074087..6f132eb 100644 --- a/.gitignore +++ b/.gitignore @@ -63,4 +63,4 @@ dist/ # misc .DS_Store -.vscode/settings.json \ No newline at end of file +.vscode/settings.json diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts index 3d10994..a639629 100644 --- a/src/jobs/ega/egaClient.ts +++ b/src/jobs/ega/egaClient.ts @@ -23,6 +23,17 @@ import { getAppConfig } from '../../config'; import getAppSecrets from '../../secrets'; import { EGA_GRANT_TYPE, EGA_REALMS_PATH, EGA_TOKEN_ENDPOINT } from '../../utils/constants'; +type IdpToken = { + access_token: string; + scope: string; + session_state: string; + token_type: 'Bearer'; + refresh_token: string; + refresh_expires_in: number; + expires_in: number; + 'not-before-policy': 0; +}; + // initialize idp client const initIdpClient = () => { const { @@ -41,6 +52,9 @@ const initApiAxiosClient = () => { } = getAppConfig(); return axios.create({ baseURL: apiUrl, + headers: { + 'Content-Type': 'application/json', + }, }); }; const apiAxiosClient = initApiAxiosClient(); @@ -49,7 +63,7 @@ const apiAxiosClient = initApiAxiosClient(); * POST request to retrieve an accessToken for the EGA API client * @returns Promise */ -const getAccessToken = async (): Promise => { +const getAccessToken = async (): Promise => { const { ega: { authRealmName, clientId }, } = getAppConfig(); @@ -77,10 +91,65 @@ const getAccessToken = async (): Promise => { return token; }; +const refreshAccessToken = async (token: IdpToken): Promise => { + const { + ega: { authRealmName, clientId }, + } = getAppConfig(); + + const response = await idpClient.post( + urlJoin(EGA_REALMS_PATH, authRealmName, EGA_TOKEN_ENDPOINT), + { + grant_type: 'refresh_token', + client_id: clientId, + refresh_token: token.refresh_token, + }, + { + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + }, + ); + console.log('Refresh response: ', response.status); + return response.data; +}; + /** * Fetches access token and attaches to Axios instance headers for apiClient * @returns API functions that use authenticated Axios instance */ export const egaApiClient = async () => { const token = await getAccessToken(); + + apiAxiosClient.defaults.headers.common['Authorization'] = `Bearer ${token.access_token}`; + + apiAxiosClient.interceptors.response.use( + (response) => response, + async (error) => { + console.log('Got here, error is ', error); + if (error.response && error.response.status === 401) { + console.log('Access expired, attempting refresh'); + // Access token has expired, refresh it + try { + const newAccessToken = await refreshAccessToken(token); + // Update the request headers with the new access token + error.config.headers['Authorization'] = `Bearer ${newAccessToken.access_token}`; + // Retry the original request + return apiAxiosClient(error.config); + } catch (refreshError) { + console.log('Refresh error: ', refreshError); + // Handle token refresh error + throw refreshError; + } + } + console.log('General error: ', error); + return Promise.reject(error); + }, + ); + + const getDacs = async () => { + const response = await apiAxiosClient.get('/dacs'); + return response.data; + }; + + return { getDacs }; }; From cfa6d35c274512321ffc0e1f7ce75833df67387a Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Mon, 23 Sep 2024 08:22:35 -0400 Subject: [PATCH 04/25] add ega endpoint constants --- src/utils/constants.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 40dfff2..95ea089 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -272,7 +272,18 @@ export const ICGC_ARGO_CONTACT_URL = urlJoin(ICGC_ARGO_PLATFORM_URL, 'contact'); export const NOTIFICATION_UNIT_OF_TIME: unitOfTime.DurationConstructor = 'days'; export const REQUEST_CHUNK_SIZE = 5; -// ega +// ega idp export const EGA_REALMS_PATH = 'realms'; export const EGA_TOKEN_ENDPOINT = 'protocol/openid-connect/token'; export const EGA_GRANT_TYPE = 'password'; + +// ega api + +export const EGA_API = { + DACS: 'dacs', + PERMISSIONS: 'permissions', + USERS: 'users', + MEMBERS: 'members', + REQUESTS: 'requests', + DATASETS: 'datasets', +}; From ce6d8a7c87ee84f227222d21c2c7a7c02c7e1dbb Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Mon, 23 Sep 2024 08:37:07 -0400 Subject: [PATCH 05/25] add section for env vars in readme --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index f1cb2dc..3b98681 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,17 @@ Development of the Data Access Control API | ---------- | ------------- | ----------------------------------------------------------------------------------------- | --------------------------------- | ------- | | `NODE_ENV` | isDevelopment | Enables `'/applications/:id'` DELETE endpoint. Enables `debug.log` file in Logger options | set `NODE_ENV` to `"development"` | `false` | +## Environment Variables + +| Name | Description | Type | Required | Default | +| ------------------- | ----------------------------------------------------------------------------- | -------- | -------- | ------- | +| EGA_CLIENT_ID | Client ID for EGA API | `string` | true | | +| EGA_AUTH_HOST | Root URL for EGA authentication server | `string` | true | | +| EGA_AUTH_REALM_NAME | Realm name for EGA authentication server | `string` | true | | +| EGA_API_URL | Root URL for EGA API | `string` | true | | +| EGA_USERNAME | Username for account used to gain access token from EGA authentication server | `string` | true | | +| EGA_PASSWORD | Password for account used to gain access token from EGA authentication server | `string` | true | | + ## Feature Flags | Name | Config Path | Description | Trigger | Default | From 912e16a0556315d3f13307b254151e8efb8c4b5e Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Mon, 23 Sep 2024 08:39:55 -0400 Subject: [PATCH 06/25] remove logs --- src/jobs/ega/egaClient.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts index a639629..66d61b4 100644 --- a/src/jobs/ega/egaClient.ts +++ b/src/jobs/ega/egaClient.ts @@ -22,6 +22,7 @@ import urlJoin from 'url-join'; import { getAppConfig } from '../../config'; import getAppSecrets from '../../secrets'; import { EGA_GRANT_TYPE, EGA_REALMS_PATH, EGA_TOKEN_ENDPOINT } from '../../utils/constants'; +import logger from '../../logger'; type IdpToken = { access_token: string; @@ -109,7 +110,7 @@ const refreshAccessToken = async (token: IdpToken): Promise => { }, }, ); - console.log('Refresh response: ', response.status); + return response.data; }; @@ -125,9 +126,8 @@ export const egaApiClient = async () => { apiAxiosClient.interceptors.response.use( (response) => response, async (error) => { - console.log('Got here, error is ', error); if (error.response && error.response.status === 401) { - console.log('Access expired, attempting refresh'); + logger.info('Access expired, attempting refresh'); // Access token has expired, refresh it try { const newAccessToken = await refreshAccessToken(token); @@ -136,12 +136,10 @@ export const egaApiClient = async () => { // Retry the original request return apiAxiosClient(error.config); } catch (refreshError) { - console.log('Refresh error: ', refreshError); // Handle token refresh error throw refreshError; } } - console.log('General error: ', error); return Promise.reject(error); }, ); From 83be068537f586992ecabf419e0be591e2b63774 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Mon, 23 Sep 2024 08:55:48 -0400 Subject: [PATCH 07/25] remove public key reference --- src/secrets.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/secrets.ts b/src/secrets.ts index 173a022..2706593 100644 --- a/src/secrets.ts +++ b/src/secrets.ts @@ -2,7 +2,6 @@ import * as dotenv from 'dotenv'; import logger from './logger'; import * as vault from './vault'; -import { fetchPublicKeyFromRealm } from './jobs/ega/publicKey'; export interface MongoSecrets { dbUser: string; @@ -22,7 +21,6 @@ export interface AppSecrets { dacoEncryptionKey: string; egaUsername: string; egaPassword: string; - egaPublicKey: string; }; storage: { key: string; @@ -52,7 +50,6 @@ const loadVaultSecrets = async () => { const buildSecrets = async (vaultSecrets: Record = {}): Promise => { logger.info('Building app secrets...'); - const publicKey = await fetchPublicKeyFromRealm(); secrets = { email: { auth: { @@ -64,7 +61,6 @@ const buildSecrets = async (vaultSecrets: Record = {}): Promise Date: Tue, 24 Sep 2024 10:37:08 -0400 Subject: [PATCH 08/25] Add zod, update ts. Add expiry to approved app list data --- package-lock.json | 326 ++-------------------- package.json | 5 +- src/domain/interface.ts | 5 +- src/domain/service/applications/search.ts | 2 + src/utils/jwt.ts | 2 +- 5 files changed, 31 insertions(+), 309 deletions(-) diff --git a/package-lock.json b/package-lock.json index 87eeb93..0d2002e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,7 +52,8 @@ "uuid": "^8.3.2", "validate.js": "^0.13.1", "winston": "^3.3.3", - "yamljs": "^0.3.0" + "yamljs": "^0.3.0", + "zod": "^3.23.8" }, "devDependencies": { "@types/bcrypt-nodejs": "^0.0.31", @@ -106,7 +107,7 @@ "sinon": "^10.0.0", "testcontainers": "^7.8.0", "ts-node-dev": "^2.0.0", - "typescript": "^4.9.5" + "typescript": "^5.6.2" } }, "node_modules/@aws-crypto/crc32": { @@ -882,14 +883,6 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.2.0.tgz", "integrity": "sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w==" }, - "node_modules/@aws-sdk/middleware-retry/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/@aws-sdk/middleware-sdk-s3": { "version": "3.18.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.18.0.tgz", @@ -1143,98 +1136,6 @@ "node": ">= 10.0.0" } }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@aws-sdk/is-array-buffer": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/is-array-buffer/-/is-array-buffer-3.18.0.tgz", - "integrity": "sha512-HvPRgESVQt0UbzRQZVKhf8SpGGc5Jrln3AtTzkVu6PBHO04Dh2EHsrsxiu7X3oB453Mnp8+LYBVIgsmM/RyJzA==", - "dependencies": { - "tslib": "^2.0.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@aws-sdk/middleware-stack": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-stack/-/middleware-stack-3.18.0.tgz", - "integrity": "sha512-+FDsKMRq3Gsd6ddVt1P+7ltSiRRcEj6KpRccMHkFkFqWWqn9OcPh+Et076ivSBXCW8q9Ib4qJi04hiCD/md2EQ==", - "dependencies": { - "tslib": "^2.0.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@aws-sdk/protocol-http": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/protocol-http/-/protocol-http-3.18.0.tgz", - "integrity": "sha512-GIKvZBEnm87/mRaVYHnsQDYBSvU6qyKjyVdHDpQHhF+MZ+MKafygmpdBjsrRRstWr7h5WepnUVImYgvmaW6vyw==", - "dependencies": { - "@aws-sdk/types": "3.18.0", - "tslib": "^2.0.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@aws-sdk/signature-v4": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4/-/signature-v4-3.18.0.tgz", - "integrity": "sha512-md52+v+aIDfhwtaN+xIJ+7XgSqtRmreGkSCnJziGINRSnUSdycoR/ZJhT5d9TbMpYHdoT0Rm9RXNXImlfKCNGw==", - "dependencies": { - "@aws-sdk/is-array-buffer": "3.18.0", - "@aws-sdk/types": "3.18.0", - "@aws-sdk/util-hex-encoding": "3.18.0", - "@aws-sdk/util-uri-escape": "3.18.0", - "tslib": "^2.0.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@aws-sdk/smithy-client": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/smithy-client/-/smithy-client-3.18.0.tgz", - "integrity": "sha512-fIcfzrf2TnhB4W8UyqdPQ9fPAfIfuLQ0dO/Y9qwzsw0Bvj4qYYPcUaNI2raX7WN1G2KHa9wZdiceR0J+uQO7yg==", - "dependencies": { - "@aws-sdk/middleware-stack": "3.18.0", - "@aws-sdk/types": "3.18.0", - "tslib": "^2.0.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@aws-sdk/types": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.18.0.tgz", - "integrity": "sha512-fyk6HXK1wk83n4fDvsG+ewV+yS4uegepeMNrmLr7iBKjzc/bLckTWk7GKFM5ZaF/9jWyk7o2eKW3C3BltgDrfQ==", - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@aws-sdk/util-hex-encoding": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-hex-encoding/-/util-hex-encoding-3.18.0.tgz", - "integrity": "sha512-tayCN0+jLJRyM7W059ybwaEojjI4ylP4UyyG+LDc4m62PskmsCWTWOJzudjtx4d765e0I/F1w1ELrE+VhUdOpQ==", - "dependencies": { - "tslib": "^2.0.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@aws-sdk/util-uri-escape": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-uri-escape/-/util-uri-escape-3.18.0.tgz", - "integrity": "sha512-Ui+uydvhzQALj/Q8sat4cVnCedwB/8iBPoMzcm1hr1r7ttWfmBKKElFZFl6ljCUtKaCE3rTb3JrZ2sKy9wT09A==", - "dependencies": { - "tslib": "^2.0.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, "node_modules/@aws-sdk/s3-request-presigner/node_modules/tslib": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.2.0.tgz", @@ -1433,38 +1334,6 @@ "node": ">= 10.0.0" } }, - "node_modules/@aws-sdk/util-create-request/node_modules/@aws-sdk/middleware-stack": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-stack/-/middleware-stack-3.18.0.tgz", - "integrity": "sha512-+FDsKMRq3Gsd6ddVt1P+7ltSiRRcEj6KpRccMHkFkFqWWqn9OcPh+Et076ivSBXCW8q9Ib4qJi04hiCD/md2EQ==", - "dependencies": { - "tslib": "^2.0.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/util-create-request/node_modules/@aws-sdk/smithy-client": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/smithy-client/-/smithy-client-3.18.0.tgz", - "integrity": "sha512-fIcfzrf2TnhB4W8UyqdPQ9fPAfIfuLQ0dO/Y9qwzsw0Bvj4qYYPcUaNI2raX7WN1G2KHa9wZdiceR0J+uQO7yg==", - "dependencies": { - "@aws-sdk/middleware-stack": "3.18.0", - "@aws-sdk/types": "3.18.0", - "tslib": "^2.0.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/util-create-request/node_modules/@aws-sdk/types": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.18.0.tgz", - "integrity": "sha512-fyk6HXK1wk83n4fDvsG+ewV+yS4uegepeMNrmLr7iBKjzc/bLckTWk7GKFM5ZaF/9jWyk7o2eKW3C3BltgDrfQ==", - "engines": { - "node": ">= 10.0.0" - } - }, "node_modules/@aws-sdk/util-create-request/node_modules/tslib": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.2.0.tgz", @@ -1483,38 +1352,6 @@ "node": ">= 10.0.0" } }, - "node_modules/@aws-sdk/util-format-url/node_modules/@aws-sdk/querystring-builder": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/querystring-builder/-/querystring-builder-3.18.0.tgz", - "integrity": "sha512-1DrzflLp80RG674XfhZsl4jehIe0mdSPqXqMH6vOMDcmF/lLEsfwPs307G+Go3kwWXSUup52bcMmfi8Ef4xLBg==", - "dependencies": { - "@aws-sdk/types": "3.18.0", - "@aws-sdk/util-uri-escape": "3.18.0", - "tslib": "^2.0.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/util-format-url/node_modules/@aws-sdk/types": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.18.0.tgz", - "integrity": "sha512-fyk6HXK1wk83n4fDvsG+ewV+yS4uegepeMNrmLr7iBKjzc/bLckTWk7GKFM5ZaF/9jWyk7o2eKW3C3BltgDrfQ==", - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/util-format-url/node_modules/@aws-sdk/util-uri-escape": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-uri-escape/-/util-uri-escape-3.18.0.tgz", - "integrity": "sha512-Ui+uydvhzQALj/Q8sat4cVnCedwB/8iBPoMzcm1hr1r7ttWfmBKKElFZFl6ljCUtKaCE3rTb3JrZ2sKy9wT09A==", - "dependencies": { - "tslib": "^2.0.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, "node_modules/@aws-sdk/util-format-url/node_modules/tslib": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.2.0.tgz", @@ -8572,9 +8409,9 @@ "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" }, "node_modules/ts-node": { - "version": "10.9.1", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", - "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -8751,15 +8588,15 @@ } }, "node_modules/typescript": { - "version": "4.9.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", - "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz", + "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=4.2.0" + "node": ">=14.17" } }, "node_modules/uglify-js": { @@ -9466,9 +9303,9 @@ } }, "node_modules/zod": { - "version": "3.19.1", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.19.1.tgz", - "integrity": "sha512-LYjZsEDhCdYET9ikFu6dVPGp2YH9DegXjdJToSzD9rO6fy4qiRYFoyEYwps88OseJlPyl2NOe2iJuhEhL7IpEA==", + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -10230,11 +10067,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.2.0.tgz", "integrity": "sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w==" - }, - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" } } }, @@ -10475,74 +10307,6 @@ "tslib": "^2.0.0" }, "dependencies": { - "@aws-sdk/is-array-buffer": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/is-array-buffer/-/is-array-buffer-3.18.0.tgz", - "integrity": "sha512-HvPRgESVQt0UbzRQZVKhf8SpGGc5Jrln3AtTzkVu6PBHO04Dh2EHsrsxiu7X3oB453Mnp8+LYBVIgsmM/RyJzA==", - "requires": { - "tslib": "^2.0.0" - } - }, - "@aws-sdk/middleware-stack": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-stack/-/middleware-stack-3.18.0.tgz", - "integrity": "sha512-+FDsKMRq3Gsd6ddVt1P+7ltSiRRcEj6KpRccMHkFkFqWWqn9OcPh+Et076ivSBXCW8q9Ib4qJi04hiCD/md2EQ==", - "requires": { - "tslib": "^2.0.0" - } - }, - "@aws-sdk/protocol-http": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/protocol-http/-/protocol-http-3.18.0.tgz", - "integrity": "sha512-GIKvZBEnm87/mRaVYHnsQDYBSvU6qyKjyVdHDpQHhF+MZ+MKafygmpdBjsrRRstWr7h5WepnUVImYgvmaW6vyw==", - "requires": { - "@aws-sdk/types": "3.18.0", - "tslib": "^2.0.0" - } - }, - "@aws-sdk/signature-v4": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4/-/signature-v4-3.18.0.tgz", - "integrity": "sha512-md52+v+aIDfhwtaN+xIJ+7XgSqtRmreGkSCnJziGINRSnUSdycoR/ZJhT5d9TbMpYHdoT0Rm9RXNXImlfKCNGw==", - "requires": { - "@aws-sdk/is-array-buffer": "3.18.0", - "@aws-sdk/types": "3.18.0", - "@aws-sdk/util-hex-encoding": "3.18.0", - "@aws-sdk/util-uri-escape": "3.18.0", - "tslib": "^2.0.0" - } - }, - "@aws-sdk/smithy-client": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/smithy-client/-/smithy-client-3.18.0.tgz", - "integrity": "sha512-fIcfzrf2TnhB4W8UyqdPQ9fPAfIfuLQ0dO/Y9qwzsw0Bvj4qYYPcUaNI2raX7WN1G2KHa9wZdiceR0J+uQO7yg==", - "requires": { - "@aws-sdk/middleware-stack": "3.18.0", - "@aws-sdk/types": "3.18.0", - "tslib": "^2.0.0" - } - }, - "@aws-sdk/types": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.18.0.tgz", - "integrity": "sha512-fyk6HXK1wk83n4fDvsG+ewV+yS4uegepeMNrmLr7iBKjzc/bLckTWk7GKFM5ZaF/9jWyk7o2eKW3C3BltgDrfQ==" - }, - "@aws-sdk/util-hex-encoding": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-hex-encoding/-/util-hex-encoding-3.18.0.tgz", - "integrity": "sha512-tayCN0+jLJRyM7W059ybwaEojjI4ylP4UyyG+LDc4m62PskmsCWTWOJzudjtx4d765e0I/F1w1ELrE+VhUdOpQ==", - "requires": { - "tslib": "^2.0.0" - } - }, - "@aws-sdk/util-uri-escape": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-uri-escape/-/util-uri-escape-3.18.0.tgz", - "integrity": "sha512-Ui+uydvhzQALj/Q8sat4cVnCedwB/8iBPoMzcm1hr1r7ttWfmBKKElFZFl6ljCUtKaCE3rTb3JrZ2sKy9wT09A==", - "requires": { - "tslib": "^2.0.0" - } - }, "tslib": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.2.0.tgz", @@ -10731,29 +10495,6 @@ "tslib": "^2.0.0" }, "dependencies": { - "@aws-sdk/middleware-stack": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-stack/-/middleware-stack-3.18.0.tgz", - "integrity": "sha512-+FDsKMRq3Gsd6ddVt1P+7ltSiRRcEj6KpRccMHkFkFqWWqn9OcPh+Et076ivSBXCW8q9Ib4qJi04hiCD/md2EQ==", - "requires": { - "tslib": "^2.0.0" - } - }, - "@aws-sdk/smithy-client": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/smithy-client/-/smithy-client-3.18.0.tgz", - "integrity": "sha512-fIcfzrf2TnhB4W8UyqdPQ9fPAfIfuLQ0dO/Y9qwzsw0Bvj4qYYPcUaNI2raX7WN1G2KHa9wZdiceR0J+uQO7yg==", - "requires": { - "@aws-sdk/middleware-stack": "3.18.0", - "@aws-sdk/types": "3.18.0", - "tslib": "^2.0.0" - } - }, - "@aws-sdk/types": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.18.0.tgz", - "integrity": "sha512-fyk6HXK1wk83n4fDvsG+ewV+yS4uegepeMNrmLr7iBKjzc/bLckTWk7GKFM5ZaF/9jWyk7o2eKW3C3BltgDrfQ==" - }, "tslib": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.2.0.tgz", @@ -10771,29 +10512,6 @@ "tslib": "^2.0.0" }, "dependencies": { - "@aws-sdk/querystring-builder": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/querystring-builder/-/querystring-builder-3.18.0.tgz", - "integrity": "sha512-1DrzflLp80RG674XfhZsl4jehIe0mdSPqXqMH6vOMDcmF/lLEsfwPs307G+Go3kwWXSUup52bcMmfi8Ef4xLBg==", - "requires": { - "@aws-sdk/types": "3.18.0", - "@aws-sdk/util-uri-escape": "3.18.0", - "tslib": "^2.0.0" - } - }, - "@aws-sdk/types": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.18.0.tgz", - "integrity": "sha512-fyk6HXK1wk83n4fDvsG+ewV+yS4uegepeMNrmLr7iBKjzc/bLckTWk7GKFM5ZaF/9jWyk7o2eKW3C3BltgDrfQ==" - }, - "@aws-sdk/util-uri-escape": { - "version": "3.18.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-uri-escape/-/util-uri-escape-3.18.0.tgz", - "integrity": "sha512-Ui+uydvhzQALj/Q8sat4cVnCedwB/8iBPoMzcm1hr1r7ttWfmBKKElFZFl6ljCUtKaCE3rTb3JrZ2sKy9wT09A==", - "requires": { - "tslib": "^2.0.0" - } - }, "tslib": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.2.0.tgz", @@ -16570,9 +16288,9 @@ "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" }, "ts-node": { - "version": "10.9.1", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", - "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "requires": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -16692,9 +16410,9 @@ } }, "typescript": { - "version": "4.9.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", - "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==" + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz", + "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==" }, "uglify-js": { "version": "3.13.9", @@ -17238,9 +16956,9 @@ } }, "zod": { - "version": "3.19.1", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.19.1.tgz", - "integrity": "sha512-LYjZsEDhCdYET9ikFu6dVPGp2YH9DegXjdJToSzD9rO6fy4qiRYFoyEYwps88OseJlPyl2NOe2iJuhEhL7IpEA==" + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==" } } } diff --git a/package.json b/package.json index 92c70df..3b8fdab 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,7 @@ "sinon": "^10.0.0", "testcontainers": "^7.8.0", "ts-node-dev": "^2.0.0", - "typescript": "^4.9.5" + "typescript": "^5.6.2" }, "dependencies": { "@aws-sdk/client-s3": "^3.18.0", @@ -123,7 +123,8 @@ "uuid": "^8.3.2", "validate.js": "^0.13.1", "winston": "^3.3.3", - "yamljs": "^0.3.0" + "yamljs": "^0.3.0", + "zod": "^3.23.8" }, "husky": { "hooks": { diff --git a/src/domain/interface.ts b/src/domain/interface.ts index 7fd995a..1c07f3f 100644 --- a/src/domain/interface.ts +++ b/src/domain/interface.ts @@ -368,12 +368,13 @@ export interface IRequest extends Request { identity: Identity; } -export interface UserDataFromApprovedApplicationsResult { +export type UserDataFromApprovedApplicationsResult = { applicant: Sections['applicant']; collaborators: Sections['collaborators']; lastUpdatedAtUtc?: Date; appId: string; -} + expiresAtUtc: Date; +}; export interface ApprovedUserRowData { userName: string; diff --git a/src/domain/service/applications/search.ts b/src/domain/service/applications/search.ts index 2584bea..937d7ee 100644 --- a/src/domain/service/applications/search.ts +++ b/src/domain/service/applications/search.ts @@ -347,6 +347,7 @@ export const getUsersFromApprovedApps = async (): Promise< 'sections.applicant': 1, 'sections.collaborators': 1, lastUpdatedAtUtc: 1, + expiresAtUtc: 1, }).exec(); return results.map((result) => { @@ -355,6 +356,7 @@ export const getUsersFromApprovedApps = async (): Promise< collaborators: result.sections.collaborators, appId: result.appId, lastUpdatedAtUtc: result.lastUpdatedAtUtc, + expiresAtUtc: result.expiresAtUtc, }; return approvedUsersInfo; }); diff --git a/src/utils/jwt.ts b/src/utils/jwt.ts index 4be248e..8a07267 100644 --- a/src/utils/jwt.ts +++ b/src/utils/jwt.ts @@ -1,5 +1,5 @@ // sample jwts for local testing based on the keys below -export const userJwt = `eyJhbGciOiJSUzI1NiJ9.eyJpYXQiOjE2MTk4MjU1NjUsImV4cCI6MTcwMTMwODc4MCwic3ViIjoiNDlmZWEyZDMtYmE0MS00OGQ5LWFjMjgtMDUxN2RhYzMwOWEyIiwiaXNzIjoiZWdvIiwianRpIjoiOGE2YzIwMWYtMzBmOS00ZmU5LTkzNjItNzdkOGZlMmZkYTk2IiwiY29udGV4dCI6eyJzY29wZSI6WyIqLkRFTlkiXSwidXNlciI6eyJlbWFpbCI6ImFwcGxpY2FudEBvaWNyLm9uLmNhIiwic3RhdHVzIjoiQVBQUk9WRUQiLCJmaXJzdE5hbWUiOiJhcHBsaSIsImxhc3ROYW1lIjoiY2FudCIsImNyZWF0ZWRBdCI6MTU4MzM0MjI5MTc0NSwibGFzdExvZ2luIjoxNjE5ODI1NTY1NjUxLCJwcmVmZXJyZWRMYW5ndWFnZSI6IkVOR0xJU0giLCJ0eXBlIjoiVVNFUiIsInByb3ZpZGVyVHlwZSI6IkdPT0dMRSIsInByb3ZpZGVyU3ViamVjdElkIjoiYXBwbGljYW50MTIzNCIsImdyb3VwcyI6WyI2NTI0Il19fSwiYXVkIjpbXX0.QBXpq0954YPnX4HUsRblBfaR0eY0HvprBN72IDPq3oaqHA2iG8cmjXMP-bj3KQPDdVbMaoCj7DRik7Zff-rvTrPAY_epjVqz8VOdd_fAhcXMj4b4MC3Zuc2-0l8Q8uXWHvUfERBW58XIF-IYCLsVHuopkn3s4YmRl7VM0dbqHr5c4Fv9gMSZP3oiD3zlpix-7WpQ2RSMfjQMul6rEDyt113q5t4OLV8d85Z9zUo4sfbhdoVig59IA9Y_9FDuVf274phfzF8v1IIs8prDcQqbNzqQ1fEqsZNEPuZ5x29cy8oMCTBXTboD_UdDvTFm1CouuUHXMFMPOuNERSl5qKu32A`; +export const userJwt = `eyJhbGciOiJSUzI1NiJ9.eyJpYXQiOjE2MTk4MjU1NjUsImV4cCI6MTc5MDIxMTgwNiwic3ViIjoiNDlmZWEyZDMtYmE0MS00OGQ5LWFjMjgtMDUxN2RhYzMwOWEyIiwiaXNzIjoiZWdvIiwianRpIjoiOGE2YzIwMWYtMzBmOS00ZmU5LTkzNjItNzdkOGZlMmZkYTk2IiwiY29udGV4dCI6eyJzY29wZSI6WyIqLkRFTlkiXSwidXNlciI6eyJlbWFpbCI6ImFwcGxpY2FudEBvaWNyLm9uLmNhIiwic3RhdHVzIjoiQVBQUk9WRUQiLCJmaXJzdE5hbWUiOiJhcHBsaSIsImxhc3ROYW1lIjoiY2FudCIsImNyZWF0ZWRBdCI6MTU4MzM0MjI5MTc0NSwibGFzdExvZ2luIjoxNjE5ODI1NTY1NjUxLCJwcmVmZXJyZWRMYW5ndWFnZSI6IkVOR0xJU0giLCJ0eXBlIjoiVVNFUiIsInByb3ZpZGVyVHlwZSI6IkdPT0dMRSIsInByb3ZpZGVyU3ViamVjdElkIjoiYXBwbGljYW50MTIzNCIsImdyb3VwcyI6WyI2NTI0Il19fSwiYXVkIjpbXX0.jOCmon8NLgI2wXxWFFjS-utJVKiPr3QtAgxmiHUPnAZP2Eal9KdPujda3dsZOJIrXaoppAUMbaSFdJSO9Xi1P254bZmAdKuMJFQgDRLwaJKK50tPK-GJviWazsNWJ1AHko70vlehxETeMSv7yqaIbu3zFK_cLQYPsCSCoEmuxsEXOkMdlwUaRqGHtMaMuKyhFas2rs_zmkjbPkRiZx-AfaUPZsF-gCcYe1lKM5CTfKQt75ebqEXUYp1CCq3qeuYoGTEslC-qkyBOsL1B9RuDOMZkOs9TY9A4-V8qGO1ySB4kbaiJa5TEvOPTq8bsQKdA52AhQTjcaHN07jYQhPuadA`; export const reviewerJwt = `eyJhbGciOiJSUzI1NiJ9.eyJpYXQiOjE2MTk4MjU1NjUsImV4cCI6MTc0ODU0MDQxMCwic3ViIjoiYWRtaW4xMjM0NSIsImlzcyI6ImVnbyIsImp0aSI6IjhhNmMyMDFmLTMwZjktNGZlOS05MzYyLTc3ZDhmZTJmZGE5NiIsImNvbnRleHQiOnsic2NvcGUiOlsiREFDTy1SRVZJRVcuV1JJVEUiLCJEQUNPLVJFVklFVy5SRUFEIl0sInVzZXIiOnsiZW1haWwiOiJiYWxsYWJhZGlAb2ljci5vbi5jYSIsInN0YXR1cyI6IkFQUFJPVkVEIiwiZmlyc3ROYW1lIjoiQmFzaGFyIiwibGFzdE5hbWUiOiJBbGxhYmFkaSIsImNyZWF0ZWRBdCI6MTU4MzM0MjI5MTc0NSwibGFzdExvZ2luIjoxNjE5ODI1NTY1NjUxLCJwcmVmZXJyZWRMYW5ndWFnZSI6IkZSRU5DSCIsInByb3ZpZGVyVHlwZSI6IkdPT0dMRSIsInByb3ZpZGVyU3ViamVjdElkIjoiZ29vZ2xlMTIyMzM0IiwidHlwZSI6IkFETUlOIiwiZ3JvdXBzIjpbIjY1MjQiXX19LCJhdWQiOltdfQ.Yne_TnFvEbkq5YzZtDiDCBdqYoB83sQMy8DOexKPJdsRpM5xfrZ7UMVnqcnQY_EV8sVAtStvwWPa5XRFDcNlWM3SJg7mBebueUJRqyYrgoyNIOl7IeQy0TOtLnuhRCojmDVvrH_HI1F9gl0DCtyFvjCgkS0RAXZ8PDtFBdV7O0uu2iK3Lw_t8NRhr6N3swl3xxGIKk5b_C2nrpgaCEI4qGYqLh9hLrYQcKEM_g2DKvSmvzkySYijquFCkxCESIVQvLhkrgM3j3zKcXD0qz9hlrKqElhS3-DidAay5uPRBT2Tz130Ub1_zm_voox9ixux4S1UgPfaRErNgEkX3Cp-YQ`; export const systemJwt = `eyJraWQiOiIyODc5Y2FiOC0zNWFiLTRlMDgtYmYzZS1kNzY4ZTcyYThiM2YiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIwMDA5OTgiLCJuYmYiOjE2NjkyMjMxNTgsInNjb3BlIjpbIkRBQ08tU1lTVEVNLldSSVRFIl0sImlzcyI6ImVnbyIsImNvbnRleHQiOnsic2NvcGUiOlsiREFDTy1TWVNURU0uV1JJVEUiLCJEQUNPLVNZU1RFTS5SRUFEIl0sImFwcGxpY2F0aW9uIjp7Im5hbWUiOiJEQUMtQVBQIiwiY2xpZW50SWQiOiJkYWMtYXBwIiwicmVkaXJlY3RVcmkiOiJyZWRpcmVjdCIsInN0YXR1cyI6IkFQUFJPVkVEIiwiZXJyb3JSZWRpcmVjdFVyaSI6ImVycm9yIiwidHlwZSI6IkNMSUVOVCJ9fSwiZXhwIjoxNzQ4NTQwNDEwLCJpYXQiOjE2NjkyMjMxNTgsImp0aSI6IjE0Yjc5NjgxLWFjNGEtNGU1Yi05NDU5LTBmYzYwOTVhY2NjZCJ9.KcpR3D6a0Q3D-tUcY9KiFN5THctAn8TShcpObdoaRSCmJSjcscMeY9hdmzmO-_XgPFKPdLC1dSRV7ZIq7FUQTrv3lZGflK_9fVMdW9YtuJy9XlsHKF5zwKZ0FUx6Qd0Ib1blD2THn8a-HyH2TWYblmTsE8mLBVmiuZc4Bfv62H-aTPfKSVYeRh7BBK-Jb2BBIKIkFW28noPXQwQK9Pv9iyWC04CnvqItKD3Ad3SLoRBSXporHLGRkfwygQ8EuusTp2zSpwIB6gg_zalmuwUKegpOLqCZUfq_Kk5iJLYnCZVNwdouT-pgkQ6hgjR208SczaZhPlKxh7Tic-d8gNYnkg`; export const readOnlyReviewerJwt = `eyJhbGciOiJSUzI1NiJ9.eyJpYXQiOjE2MTk4MjU1NjUsImV4cCI6MTc0ODU0MDQxMCwic3ViIjoicm9hZG1pbjEyMzQ1IiwiaXNzIjoiZWdvIiwianRpIjoiOGE2YzIwMWYtMzBmOS00ZmU5LTkzNjItNzdkOGZlMmZkYTk2IiwiY29udGV4dCI6eyJzY29wZSI6WyJEQUNPLVJFVklFVy5SRUFEIl0sInVzZXIiOnsiZW1haWwiOiJyZWFkT25seUFkbWluQGV4YW1wbGUuY29tIiwic3RhdHVzIjoiQVBQUk9WRUQiLCJmaXJzdE5hbWUiOiJSZWFkIiwibGFzdE5hbWUiOiJPbmx5IiwiY3JlYXRlZEF0IjoxNTgzMzQyMjkxNzQ1LCJsYXN0TG9naW4iOjE2MTk4MjU1NjU2NTEsInByZWZlcnJlZExhbmd1YWdlIjoiRU5HTElTSCIsInByb3ZpZGVyVHlwZSI6IkdPT0dMRSIsInByb3ZpZGVyU3ViamVjdElkIjoicmVhZC1vbmx5LWdvb2dsZS0xMjMiLCJ0eXBlIjoiQURNSU4iLCJncm91cHMiOltdfX0sImF1ZCI6W119.IYmglxfg_7wPpwurY0I1J6OLFoAj1tZRLV8i4JVC06gKV9uV_iFZFkf8Jw2DE4E06N_bSWVo-ORl-gyVE0_mGfV_evAbheyOtRigEG0HgGSUzdjB8tnDJikrrJLHzaWpHaiI9gGmFUt8sm1lMtnOCykZrHQynpcBhxrI3GpqdUnAN5IS4Hrn___s2sfAYKfsVVBQCGkg_ityQajjG8QU7McYIHgC0YcIxzKQneFwkYhpD14N8OJWSw7PqDsRdawTVj3fkcu_zg1D8r-CW01cBWXpL2BF6FvdOHvzXW7zTZr3B67U2V8zOH_w9lPjkcigatgsiITJln5E6vI19EDRoQ`; From 870ba74b20c45c1dabf90915bbd34399dcf33b05 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Tue, 24 Sep 2024 10:42:26 -0400 Subject: [PATCH 09/25] Add api response types, permissions and users get funcs --- src/jobs/ega/egaClient.ts | 144 +++++++++++++++++++++++++++++++++----- src/jobs/ega/errors.ts | 10 +++ src/jobs/ega/types.ts | 108 ++++++++++++++++++++++++++++ src/jobs/ega/utils.ts | 79 +++++++++++++++++++++ 4 files changed, 325 insertions(+), 16 deletions(-) create mode 100644 src/jobs/ega/errors.ts create mode 100644 src/jobs/ega/types.ts create mode 100644 src/jobs/ega/utils.ts diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts index 66d61b4..81d4bc5 100644 --- a/src/jobs/ega/egaClient.ts +++ b/src/jobs/ega/egaClient.ts @@ -17,13 +17,25 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import axios from 'axios'; +import axios, { AxiosError, AxiosHeaders } from 'axios'; import urlJoin from 'url-join'; import { getAppConfig } from '../../config'; import getAppSecrets from '../../secrets'; -import { EGA_GRANT_TYPE, EGA_REALMS_PATH, EGA_TOKEN_ENDPOINT } from '../../utils/constants'; import logger from '../../logger'; +import { + EGA_API, + EGA_GRANT_TYPE, + EGA_REALMS_PATH, + EGA_TOKEN_ENDPOINT, +} from '../../utils/constants'; +import { EgaPermission, EgaUser } from './types'; +import { getApprovedUsers } from './utils'; +import { NotFoundError } from './errors'; + +const { DACS, PERMISSIONS, USERS } = EGA_API; +const DAC_ACCESSION_ID = 'EGAC00001000010'; + type IdpToken = { access_token: string; scope: string; @@ -96,7 +108,6 @@ const refreshAccessToken = async (token: IdpToken): Promise => { const { ega: { authRealmName, clientId }, } = getAppConfig(); - const response = await idpClient.post( urlJoin(EGA_REALMS_PATH, authRealmName, EGA_TOKEN_ENDPOINT), { @@ -126,28 +137,129 @@ export const egaApiClient = async () => { apiAxiosClient.interceptors.response.use( (response) => response, async (error) => { - if (error.response && error.response.status === 401) { - logger.info('Access expired, attempting refresh'); - // Access token has expired, refresh it - try { - const newAccessToken = await refreshAccessToken(token); - // Update the request headers with the new access token - error.config.headers['Authorization'] = `Bearer ${newAccessToken.access_token}`; - // Retry the original request - return apiAxiosClient(error.config); - } catch (refreshError) { - // Handle token refresh error - throw refreshError; + if (error instanceof AxiosError) { + if (error.response && error.response.status === 401) { + console.log('Access expired, attempting refresh'); + // Access token has expired, refresh it + try { + const newAccessToken = await refreshAccessToken(token); + // Update the request headers with the new access token + const headers = new AxiosHeaders(error.config?.headers); + headers.setAuthorization(`Bearer ${newAccessToken.access_token}`); + error.config = { + ...error.config, + headers, + }; + // Retry the original request + return apiAxiosClient(error.config); + } catch (refreshError) { + console.log('Refresh error: ', refreshError); + // Handle token refresh error + throw refreshError; + } + } + if (error.status === 404) { + throw new NotFoundError(error.message); } } return Promise.reject(error); }, ); + /** + * Retrieve a list of DACs of which the user is a member + * @returns Dac[] + * @example + * // returns [ + * { + * "provisional_id": 0, + * "accession_id": "123", + * "title": "'Dac 1'", + * "description": "Dac 1", + * "status": "accepted", + * "declined_reason": null + * } + * ] + */ const getDacs = async () => { const response = await apiAxiosClient.get('/dacs'); return response.data; }; - return { getDacs }; + /** + * Retrieve EGA user data for each user on DACO approved list + * @returns EGAUser[] + * @example + * // returns [ + * { + * id: 123, + * username: boysue@example.com, + * email: boysue@example.com, + * accession_id: EGAW00000009999 + * } + * ] + */ + const getUsers = async (): Promise => { + const dacoUsers = await getApprovedUsers(); + let egaUsers: EgaUser[] = []; + for await (const user of dacoUsers) { + try { + // TODO: handle 404 properly. If the User is not in EGA, do we add to report? Or just ignore? We can't proceed with permissions without a userId + const { data } = await apiAxiosClient.get(urlJoin(USERS, user.email)); + const egaUser = EgaUser.safeParse(data); + if (egaUser.success) { + logger.info('Successfully parsed ', user.email, '. Adding to list.'); + egaUsers.push(egaUser.data); + } + } catch (err) { + if (err instanceof AxiosError) { + switch (err.code) { + case 'NOT_FOUND': + // TODO: add user to error report? + logger.error('User not found'); + break; + default: + logger.error('Axios error'); + } + } else { + logger.error('System error'); + } + } + } + return egaUsers; + }; + + const getPermissionsForDataset = async (dataset_accession_id: string) => { + const url = urlJoin(DACS, DAC_ACCESSION_ID, PERMISSIONS); + + let results: EgaPermission[] = []; + let offset = 0; + let limit = 100; + let paging = true; + + // loop will stop once result length from GET is less than limit + while (paging) { + const permissions = await apiAxiosClient.get(url, { + params: { + dataset_accession_id, + limit, + offset, + }, + }); + // TODO: add permission to a "toDelete" list if not found in approved list + // this function will return that list to be sent to the revoke function + results.push(permissions.data); + offset = offset + 100; + paging = permissions.data.length >= limit; + console.log(results.length); + } + return results.flat(); + }; + + // TODO: add remaining API requests + const getPermissionByDatasetAndUserId = async () => {}; + const createPermissionRequests = async () => {}; + const approvePermissionRequests = async () => {}; + const revokePermissions = async () => {}; + return { getDacs, getPermissionsForDataset, getUsers }; }; diff --git a/src/jobs/ega/errors.ts b/src/jobs/ega/errors.ts new file mode 100644 index 0000000..299ce78 --- /dev/null +++ b/src/jobs/ega/errors.ts @@ -0,0 +1,10 @@ +import { AxiosError } from 'axios'; + +export class NotFoundError extends AxiosError { + constructor(message: string) { + super(message); + this.name = 'Not Found'; + this.status = 404; + this.code = 'NOT_FOUND'; + } +} diff --git a/src/jobs/ega/types.ts b/src/jobs/ega/types.ts new file mode 100644 index 0000000..3a35261 --- /dev/null +++ b/src/jobs/ega/types.ts @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { z } from 'zod'; + +// For YYYY-MM-DD Date strings (i.e. '2021-01-01') +export const DateString = z.string().date(); +export type DateString = z.infer; + +// For ISO8601 Datetime strings (i.e. '2021-01-01T00:00:00.000Z') +export const DateTime = z.string().datetime(); +export type DateTime = z.infer; + +// Enums +const DAC_STATUS_ENUM = ['accepted', 'pending', 'declined'] as const; +export const DacStatus = z.enum(DAC_STATUS_ENUM); +export type DacStatus = z.infer; + +const DAC_ACCESSION_ID_REGEX = new RegExp(`^EGAC\\d{11}$`); +export const DacAccessionId = z.string().regex(DAC_ACCESSION_ID_REGEX); +export type DacAccessionId = z.infer; + +const DATASET_ACCESSION_ID_REGEX = new RegExp(`^EGAD\\d{11}$`); +export const DatasetAccessionId = z.string().regex(DATASET_ACCESSION_ID_REGEX); +export type DatasetAccessionId = z.infer; + +const USER_ACCESSION_ID_REGEX = new RegExp(`^EGAW\\d{11}$`); +export const UserAccessionId = z.string().regex(USER_ACCESSION_ID_REGEX); +export type UserAccessionId = z.infer; + +// EGA Response Types +export const Dac = z.object({ + provisional_id: z.number(), + accession_id: z.string(), + title: z.string(), + description: z.string(), + status: DacStatus, + declined_reason: z.string().nullable(), +}); +export type Dac = z.infer; + +export const EgaUser = z.object({ + id: z.number(), + username: z.string(), + // several Users are coming back with null email values, is this expected? Assuming that if there is a userid, the User is valid + email: z.string().nullable(), + accession_id: UserAccessionId, +}); +export type EgaUser = z.infer; + +export const EgaPermissionRequest = z.object({ + request_id: z.number(), + status: z.string(), + request_data: z.object({ + comment: z.string(), + }), + // TODO: api docs state this should be a DateTime string, but receiving 'YYYY-MM-DD` string. May need to change to coerceable date? + date: DateString, + username: z.string(), + full_name: z.string(), + email: z.string().email(), + organisation: z.string(), + dataset_accession_id: DatasetAccessionId, + dataset_title: z.string().nullable(), + dac_accession_id: DacAccessionId, + dac_comment: z.string().nullable(), + dac_comment_edited_at: DateTime.nullable(), // TODO: api docs state this should be DateTime string, but need to verify +}); +export type EgaPermissionRequest = z.infer; + +export const EgaPermission = z.object({ + permission_id: z.number(), + username: z.string(), + user_accession_id: UserAccessionId, + dataset_accession_id: DatasetAccessionId, +}); +export type EgaPermission = z.infer; + +// Axios +export type Success = { + success: true; + data: T; +}; + +// Request Data Types +export type PermissionRequest = { + username: string; + dataset_accession_id: DatasetAccessionId; + request_data: { + comment: string; + }; +}; diff --git a/src/jobs/ega/utils.ts b/src/jobs/ega/utils.ts new file mode 100644 index 0000000..058a010 --- /dev/null +++ b/src/jobs/ega/utils.ts @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { getUsersFromApprovedApps } from '../../domain/service/applications/search'; +import { UserDataFromApprovedApplicationsResult } from '../../domain/interface'; +import { uniqBy } from 'lodash'; +import { DatasetAccessionId, PermissionRequest } from './types'; + +type ApprovedUser = { + username: string; + email: string; + affiliation: string; + appExpiry: Date; +}; + +/** + * Extracts fields necessary for EGA permissions flow from applicant and collaborators in an application + * @param applicationData UserDataFromApprovedApplicationsResult + * @returns ApprovedUser[] + */ +const parseApprovedUsersForApplication = ( + applicationData: UserDataFromApprovedApplicationsResult, +): ApprovedUser[] => { + const applicantInfo = applicationData.applicant.info; + const applicant = { + username: applicantInfo.displayName, + email: applicantInfo.institutionEmail, + affiliation: applicantInfo.primaryAffiliation, + appExpiry: applicationData.expiresAtUtc, + }; + const collabs = (applicationData.collaborators.list || []).map((collab) => ({ + username: collab.info.displayName, + email: collab.info.institutionEmail, + affiliation: collab.info.primaryAffiliation, + appExpiry: applicationData.expiresAtUtc, + })); + + return [applicant, ...collabs].flat(); +}; + +/** + * Retrieves applicant and collaborator information from all currently approved applications + * @returns Promise + */ +export const getApprovedUsers = async () => { + const results = await getUsersFromApprovedApps(); + const parsedUsers = results.map((app) => parseApprovedUsersForApplication(app)).flat(); + return uniqBy(parsedUsers, 'email'); +}; + +// Utils +const createPermissionRequest = ( + username: string, + dataset_accession_id: DatasetAccessionId, +): PermissionRequest => { + return { + username, + dataset_accession_id, + request_data: { + comment: 'Access granted by ICGC DAC', + }, + }; +}; From 6162104e135ada8d4244a7ca1b5aab1ea8f054bf Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Wed, 25 Sep 2024 08:53:00 -0400 Subject: [PATCH 10/25] Add safe parsing for api list results. Add dacId to config --- .env.example | 1 + README.md | 1 + src/config.ts | 2 + src/jobs/ega/egaClient.ts | 247 ++++++++++++++++++++++++++++++-------- src/jobs/ega/errors.ts | 21 +++- src/jobs/ega/types.ts | 24 ++++ src/jobs/ega/utils.ts | 49 +++++++- 7 files changed, 293 insertions(+), 52 deletions(-) diff --git a/.env.example b/.env.example index 70554ed..9c18b84 100644 --- a/.env.example +++ b/.env.example @@ -119,3 +119,4 @@ EGA_AUTH_REALM_NAME= EGA_API_URL= EGA_USERNAME= EGA_PASSWORD= +DAC_ID= \ No newline at end of file diff --git a/README.md b/README.md index 3b98681..886c486 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Development of the Data Access Control API | EGA_API_URL | Root URL for EGA API | `string` | true | | | EGA_USERNAME | Username for account used to gain access token from EGA authentication server | `string` | true | | | EGA_PASSWORD | Password for account used to gain access token from EGA authentication server | `string` | true | | +| DAC_ID | AccessionId for ICGC DAC | `string` | true | | ## Feature Flags diff --git a/src/config.ts b/src/config.ts index 4138772..939741f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -95,6 +95,7 @@ export interface AppConfig { authHost: string; authRealmName: string; apiUrl: string; + dacId: string; }; } @@ -211,6 +212,7 @@ const buildAppContext = (): AppConfig => { authHost: checkIsDefined(process.env.EGA_AUTH_HOST), authRealmName: checkIsDefined(process.env.EGA_AUTH_REALM_NAME), apiUrl: checkIsDefined(process.env.EGA_API_URL), + dacId: checkIsDefined(process.env.DAC_ID), }, }; return config; diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts index 81d4bc5..b8baf7b 100644 --- a/src/jobs/ega/egaClient.ts +++ b/src/jobs/ega/egaClient.ts @@ -20,8 +20,8 @@ import axios, { AxiosError, AxiosHeaders } from 'axios'; import urlJoin from 'url-join'; import { getAppConfig } from '../../config'; -import getAppSecrets from '../../secrets'; import logger from '../../logger'; +import getAppSecrets from '../../secrets'; import { EGA_API, @@ -29,12 +29,23 @@ import { EGA_REALMS_PATH, EGA_TOKEN_ENDPOINT, } from '../../utils/constants'; -import { EgaPermission, EgaUser } from './types'; -import { getApprovedUsers } from './utils'; import { NotFoundError } from './errors'; +import { + ApprovePermissionRequest, + ApprovePermissionResponse, + DacAccessionId, + Dataset, + DatasetAccessionId, + EgaPermission, + EgaPermissionRequest, + EgaUser, + PermissionRequest, + RevokePermission, + RevokePermissionResponse, +} from './types'; +import { getApprovedUsers, safeParseArray, ZodResultAccumulator } from './utils'; -const { DACS, PERMISSIONS, USERS } = EGA_API; -const DAC_ACCESSION_ID = 'EGAC00001000010'; +const { DACS, DATASETS, PERMISSIONS, REQUESTS, USERS } = EGA_API; type IdpToken = { access_token: string; @@ -130,6 +141,9 @@ const refreshAccessToken = async (token: IdpToken): Promise => { * @returns API functions that use authenticated Axios instance */ export const egaApiClient = async () => { + const { + ega: { dacId }, + } = getAppConfig(); const token = await getAccessToken(); apiAxiosClient.defaults.headers.common['Authorization'] = `Bearer ${token.access_token}`; @@ -167,23 +181,19 @@ export const egaApiClient = async () => { ); /** - * Retrieve a list of DACs of which the user is a member - * @returns Dac[] - * @example - * // returns [ - * { - * "provisional_id": 0, - * "accession_id": "123", - * "title": "'Dac 1'", - * "description": "Dac 1", - * "status": "accepted", - * "declined_reason": null - * } - * ] + * GET request to retrieve all currently release datasets released for a DAC + * @param dacId DacAccessionId + * @returns Dataset[] */ - const getDacs = async () => { - const response = await apiAxiosClient.get('/dacs'); - return response.data; + const getDatasetsForDac = async (dacId: DacAccessionId): Promise => { + const url = urlJoin(DACS, dacId, DATASETS); + try { + const { data } = await apiAxiosClient.get(url); + return data; + } catch (err) { + logger.error(`Error retrieving datasets for DAC ${dacId}.`); + return []; + } }; /** @@ -198,13 +208,13 @@ export const egaApiClient = async () => { * accession_id: EGAW00000009999 * } * ] + * getUser('boysue@example.com') */ const getUsers = async (): Promise => { const dacoUsers = await getApprovedUsers(); let egaUsers: EgaUser[] = []; for await (const user of dacoUsers) { try { - // TODO: handle 404 properly. If the User is not in EGA, do we add to report? Or just ignore? We can't proceed with permissions without a userId const { data } = await apiAxiosClient.get(urlJoin(USERS, user.email)); const egaUser = EgaUser.safeParse(data); if (egaUser.success) { @@ -229,37 +239,180 @@ export const egaApiClient = async () => { return egaUsers; }; - const getPermissionsForDataset = async (dataset_accession_id: string) => { - const url = urlJoin(DACS, DAC_ACCESSION_ID, PERMISSIONS); - - let results: EgaPermission[] = []; - let offset = 0; - let limit = 100; - let paging = true; - - // loop will stop once result length from GET is less than limit - while (paging) { - const permissions = await apiAxiosClient.get(url, { + /** + * GET request for list of existing permissions for a dataset + * Endpoint is paginated. + * @param datasetAccessionId: DatasetAccessionId + * @param limit number + * @param offset number + * @returns EgaPermission[] + */ + const getPermissionsForDataset = async ({ + datasetAccessionId, + limit, + offset, + }: { + datasetAccessionId: DatasetAccessionId; + limit: number; + offset: number; + }): Promise | undefined> => { + const url = urlJoin(DACS, dacId, PERMISSIONS); + try { + const { data } = await apiAxiosClient.get(url, { params: { - dataset_accession_id, + dataset_accession_id: datasetAccessionId, limit, offset, }, }); - // TODO: add permission to a "toDelete" list if not found in approved list - // this function will return that list to be sent to the revoke function - results.push(permissions.data); - offset = offset + 100; - paging = permissions.data.length >= limit; - console.log(results.length); + + const result = safeParseArray(EgaPermission, data); + return result; + } catch (err) { + logger.error(err); + return undefined; + } + }; + + /** + * GET request to retrieve existing dataset permissions for a user + * @param userId string + * @param datasetId DatasetAccessionId + * @returns EgaPermission[] + */ + const getPermissionByDatasetAndUserId = async ( + userId: string, + datasetId: DatasetAccessionId, + ): Promise => { + try { + const url = urlJoin(DACS, dacId, PERMISSIONS); + const { data } = await apiAxiosClient.get(url, { + params: { + dataset_accession_id: datasetId, + user_id: userId, + }, + }); + if (!data.length) { + return undefined; + } + const result = EgaPermission.safeParse(data[0]); + if (result.success) { + return result.data; + } + } catch (err) { + logger.error('Error retrieving permission for user'); + } + }; + + /** + * POST request to create PermissionRequests for a user + * @param requests PermissionRequest[] + * @returns EgaPermissionRequest[] + * @example + * // returns [ + * { + * "request_id": 1, + * "status": "pending", + * "request_data": { + * "comment": "I'd like to access the dataset" + * }, + * "date": "2024-01-31T16:24:13.725724+00:00", + * "username": "boysue", + * "full_name": "Boy Sue", + * "email": "boysue@example.com", + * "organisation": "Research Center", + * "dataset_accession_id": "EGAD00000000001", + * "dataset_title": "Dataset 8", + * "dac_accession_id": "EGAC00000000001", + * "dac_comment": "ticket", + * "dac_comment_edited_at": "2024-01-31T16:25:13.725724+00:00" + * } + * ] + * createPermissionRequests([{ + * username: "boysue", + * dac_accession_id: "EGAC00000000001", + * request_data: { + * "comment": "I'd like to access the dataset" + * }, + * }]) + */ + const createPermissionRequests = async ( + requests: PermissionRequest[], + ): Promise => { + try { + const { data } = await apiAxiosClient.post(REQUESTS, { + requests, + }); + return data; + } catch (err) { + logger.error('Create permissions request failed'); + return undefined; + } + }; + + /** + * Approves permissions by permission id. + * Endpoint accepts an array so multiple permissions can be approved in one request. + * @param requests + * @returns + * @example + * // returns { num_granted: 2 } + * revokePermissions( + * [ + * { request_id: 10, expires_at: "2025-01-31T16:25:13.725724+00:00" }, + * { request_id: 12, expires_at: "2026-01-31T16:25:13.725724+00:00" } + * ] + * ) + */ + const approvePermissionRequests = async ( + requests: ApprovePermissionRequest[], + ): Promise => { + try { + const { data } = await apiAxiosClient.put(REQUESTS, { + requests, + }); + return data; + } catch (err) { + logger.error('Create permissions request failed'); + return undefined; } - return results.flat(); }; - // TODO: add remaining API requests - const getPermissionByDatasetAndUserId = async () => {}; - const createPermissionRequests = async () => {}; - const approvePermissionRequests = async () => {}; - const revokePermissions = async () => {}; - return { getDacs, getPermissionsForDataset, getUsers }; + /** + * Revokes permissions by permission id. + * Endpoint accepts an array so multiple permissions can be revoke in one request. + * @param requests RevokePermission[] + * @returns RevokePermissionResponse + * @example + * // returns { num_revoked: 2 } + * revokePermissions( + * [ + * { id: 10, reason: 'Access expired' }, + * { id: 12, reason: 'Access expired' } + * ] + * ) + */ + const revokePermissions = async ( + requests: RevokePermission[], + ): Promise => { + try { + const { data } = await apiAxiosClient.delete(PERMISSIONS, { data: requests }); + return data; + } catch (err) { + logger.error('Create permissions request failed'); + return undefined; + } + }; + + return { + approvePermissionRequests, + createPermissionRequests, + getDatasetsForDac, + getPermissionByDatasetAndUserId, + getPermissionsForDataset, + getUsers, + revokePermissions, + }; }; + +export type EgaClient = Awaited>; diff --git a/src/jobs/ega/errors.ts b/src/jobs/ega/errors.ts index 299ce78..21d6397 100644 --- a/src/jobs/ega/errors.ts +++ b/src/jobs/ega/errors.ts @@ -1,9 +1,28 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + import { AxiosError } from 'axios'; export class NotFoundError extends AxiosError { constructor(message: string) { super(message); - this.name = 'Not Found'; + this.name = 'NotFound'; this.status = 404; this.code = 'NOT_FOUND'; } diff --git a/src/jobs/ega/types.ts b/src/jobs/ega/types.ts index 3a35261..7fa9876 100644 --- a/src/jobs/ega/types.ts +++ b/src/jobs/ega/types.ts @@ -55,6 +55,13 @@ export const Dac = z.object({ }); export type Dac = z.infer; +export const Dataset = z.object({ + accession_id: DatasetAccessionId, + title: z.string(), + description: z.string().optional(), +}); +export type Dataset = z.infer; + export const EgaUser = z.object({ id: z.number(), username: z.string(), @@ -89,9 +96,16 @@ export const EgaPermission = z.object({ username: z.string(), user_accession_id: UserAccessionId, dataset_accession_id: DatasetAccessionId, + dac_accession_id: DacAccessionId, }); export type EgaPermission = z.infer; +export const ApprovePermissionResponse = z.object({ num_granted: z.number() }); +export type ApprovePermissionResponse = z.infer; + +export const RevokePermissionResponse = z.object({ num_revoked: z.number() }); +export type RevokePermissionResponse = z.infer; + // Axios export type Success = { success: true; @@ -106,3 +120,13 @@ export type PermissionRequest = { comment: string; }; }; + +export type ApprovePermissionRequest = { + request_id: number; + expires_at: DateTime; +}; + +export type RevokePermission = { + id: number; + reason: string; +}; diff --git a/src/jobs/ega/utils.ts b/src/jobs/ega/utils.ts index 058a010..35a6319 100644 --- a/src/jobs/ega/utils.ts +++ b/src/jobs/ega/utils.ts @@ -17,9 +17,10 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { getUsersFromApprovedApps } from '../../domain/service/applications/search'; -import { UserDataFromApprovedApplicationsResult } from '../../domain/interface'; import { uniqBy } from 'lodash'; +import { ZodError, ZodTypeAny, z } from 'zod'; +import { UserDataFromApprovedApplicationsResult } from '../../domain/interface'; +import { getUsersFromApprovedApps } from '../../domain/service/applications/search'; import { DatasetAccessionId, PermissionRequest } from './types'; type ApprovedUser = { @@ -65,15 +66,55 @@ export const getApprovedUsers = async () => { }; // Utils + +/** + * Create Ega permission request object for POST /requests + * @param username + * @param dataset_accession_id + * @returns PermissionRequest + */ const createPermissionRequest = ( username: string, - dataset_accession_id: DatasetAccessionId, + datasetAccessionId: DatasetAccessionId, ): PermissionRequest => { return { username, - dataset_accession_id, + dataset_accession_id: datasetAccessionId, request_data: { comment: 'Access granted by ICGC DAC', }, }; }; + +export type ZodResultAccumulator = { success: T[]; failure: ZodError[] }; +/** + * Parses an array of Zod SafeParseReturnType results into success (successful parse) and failure (parsing error) + * @param acc ZodResultAccumulator + * @param item z.SafeParseReturnType + * @returns ZodResultAccumulator + */ +const resultReducer = (acc: ZodResultAccumulator, item: z.SafeParseReturnType) => { + if (item.success) { + acc.success.push(item.data); + } else { + acc.failure.push(item.error); + } + return acc; +}; + +/** + * Run Zod safeParse for Schema T on an array of items, and split results by SafeParseReturnType 'success' or 'error'. + * @params schema + * @params data unknown[] + * @returns { success: [], failure: [] } + */ +export const safeParseArray = ( + schema: T, + data: Array, +): ZodResultAccumulator> => + data + .map((i) => schema.safeParse(i)) + .reduce>>((acc, item) => resultReducer(acc, item), { + success: [], + failure: [], + }); From 3570e4a67346d046a07e2c383941a9596b4428a5 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Wed, 25 Sep 2024 09:43:29 -0400 Subject: [PATCH 11/25] add tsdocs --- src/routes/utils.ts | 37 +++++++++++++++++++++++++++++++++++++ src/utils/constants.ts | 8 +++++--- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/routes/utils.ts b/src/routes/utils.ts index 5bcb12e..07dc775 100644 --- a/src/routes/utils.ts +++ b/src/routes/utils.ts @@ -19,6 +19,24 @@ import { BadRequest } from '../utils/errors'; +/** + * Validates an id string is present and matches the expected `DACO-` format. + * Will throw a BadRequest error if either condition is not met. + * Intended for validating the :id path param for a Request + * @param id string + * @returns id string + * @example + * // returns "DACO-20" + * validateId("DACO-20") + * + * @example + * // throws BadRequest + * validateId("BAZ-2") + * + * @example + * // throws BadRequest + * validateId(undefined) + */ export function validateId(id: string) { if (!id) { throw new BadRequest('id is required'); @@ -29,6 +47,25 @@ export function validateId(id: string) { return id; } +/** + * Validates a file type request parameter against allowable types, and converts the string to uppercase if validated + * Will throw a BadRequest error if provided arg does not match any of the allow list. + * Intended for validating the "type" parameter on a Request + * @param type string + * @returns type string + * + * @example + * // returns 'ETHICS' + * validateType('ethics') + * + * @example + * // returns 'SIGNED_APP' + * validateType('SIGNED_APP') + * + * @example + * // throws BadRequest + * validateType('wrong_pdf') + */ export function validateType(type: string) { if ( !['ETHICS', 'SIGNED_APP', 'APPROVED_PDF', 'ethics', 'signed_app', 'approved_pdf'].includes(type) diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 95ea089..8a00047 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -272,13 +272,15 @@ export const ICGC_ARGO_CONTACT_URL = urlJoin(ICGC_ARGO_PLATFORM_URL, 'contact'); export const NOTIFICATION_UNIT_OF_TIME: unitOfTime.DurationConstructor = 'days'; export const REQUEST_CHUNK_SIZE = 5; -// ega idp +/** EGA IDP */ +// pathnames export const EGA_REALMS_PATH = 'realms'; export const EGA_TOKEN_ENDPOINT = 'protocol/openid-connect/token'; +// oauth grant type parameter export const EGA_GRANT_TYPE = 'password'; -// ega api - +//** EGA API */ +// pathnames export const EGA_API = { DACS: 'dacs', PERMISSIONS: 'permissions', From 76a740eb69d30eb520bbbf8635d55ba870a664d5 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Wed, 25 Sep 2024 13:57:42 -0400 Subject: [PATCH 12/25] Add safeParse checks to ega client calls. --- .env.example | 2 +- src/jobs/ega/egaClient.ts | 84 ++++++++++++++++++++++----------------- src/jobs/ega/errors.ts | 14 +++++++ src/jobs/ega/types.ts | 69 +++++++++++++++++++++++++++----- src/jobs/ega/utils.ts | 36 +---------------- 5 files changed, 124 insertions(+), 81 deletions(-) diff --git a/.env.example b/.env.example index 9c18b84..57592ce 100644 --- a/.env.example +++ b/.env.example @@ -119,4 +119,4 @@ EGA_AUTH_REALM_NAME= EGA_API_URL= EGA_USERNAME= EGA_PASSWORD= -DAC_ID= \ No newline at end of file +DAC_ID= diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts index b8baf7b..38a85ac 100644 --- a/src/jobs/ega/egaClient.ts +++ b/src/jobs/ega/egaClient.ts @@ -29,7 +29,7 @@ import { EGA_REALMS_PATH, EGA_TOKEN_ENDPOINT, } from '../../utils/constants'; -import { NotFoundError } from './errors'; +import { NotFoundError, TooManyRequestsError } from './errors'; import { ApprovePermissionRequest, ApprovePermissionResponse, @@ -39,25 +39,17 @@ import { EgaPermission, EgaPermissionRequest, EgaUser, + IdpToken, PermissionRequest, RevokePermission, RevokePermissionResponse, + safeParseArray, + ZodResultAccumulator, } from './types'; -import { getApprovedUsers, safeParseArray, ZodResultAccumulator } from './utils'; +import { ApprovedUser } from './utils'; const { DACS, DATASETS, PERMISSIONS, REQUESTS, USERS } = EGA_API; -type IdpToken = { - access_token: string; - scope: string; - session_state: string; - token_type: 'Bearer'; - refresh_token: string; - refresh_expires_in: number; - expires_in: number; - 'not-before-policy': 0; -}; - // initialize idp client const initIdpClient = () => { const { @@ -85,7 +77,7 @@ const apiAxiosClient = initApiAxiosClient(); /** * POST request to retrieve an accessToken for the EGA API client - * @returns Promise + * @returns Promise */ const getAccessToken = async (): Promise => { const { @@ -111,10 +103,19 @@ const getAccessToken = async (): Promise => { }, ); - const token = response.data; - return token; + const token = IdpToken.safeParse(response.data); + if (token.success) { + return token.data; + } + logger.error('Authentication with EGA failed.'); + throw new Error('Failed to retrieve access token'); }; +/** + * POST request to retrieve a new access token via refresh token flow + * @param token IdpToken + * @returns IdpToken + */ const refreshAccessToken = async (token: IdpToken): Promise => { const { ega: { authRealmName, clientId }, @@ -133,7 +134,12 @@ const refreshAccessToken = async (token: IdpToken): Promise => { }, ); - return response.data; + const result = IdpToken.safeParse(response.data); + if (result.success) { + return result.data; + } + logger.error('Refresh access token request failed.'); + throw new Error('Failed to refresh access token'); }; /** @@ -153,7 +159,7 @@ export const egaApiClient = async () => { async (error) => { if (error instanceof AxiosError) { if (error.response && error.response.status === 401) { - console.log('Access expired, attempting refresh'); + logger.info('Access expired, attempting refresh'); // Access token has expired, refresh it try { const newAccessToken = await refreshAccessToken(token); @@ -167,7 +173,6 @@ export const egaApiClient = async () => { // Retry the original request return apiAxiosClient(error.config); } catch (refreshError) { - console.log('Refresh error: ', refreshError); // Handle token refresh error throw refreshError; } @@ -175,21 +180,27 @@ export const egaApiClient = async () => { if (error.status === 404) { throw new NotFoundError(error.message); } + if (error.status === 429) { + throw new TooManyRequestsError(error.message); + } } - return Promise.reject(error); + return new Response('Server error', { status: 500 }); }, ); /** * GET request to retrieve all currently release datasets released for a DAC * @param dacId DacAccessionId - * @returns Dataset[] + * @returns ZodResultAccumulator */ - const getDatasetsForDac = async (dacId: DacAccessionId): Promise => { + const getDatasetsForDac = async ( + dacId: DacAccessionId, + ): Promise | []> => { const url = urlJoin(DACS, dacId, DATASETS); try { const { data } = await apiAxiosClient.get(url); - return data; + const result = safeParseArray(Dataset, data); + return result; } catch (err) { logger.error(`Error retrieving datasets for DAC ${dacId}.`); return []; @@ -198,6 +209,7 @@ export const egaApiClient = async () => { /** * Retrieve EGA user data for each user on DACO approved list + * @param dacoUsers ApprovedUser[] * @returns EGAUser[] * @example * // returns [ @@ -210,8 +222,7 @@ export const egaApiClient = async () => { * ] * getUser('boysue@example.com') */ - const getUsers = async (): Promise => { - const dacoUsers = await getApprovedUsers(); + const getUsers = async (dacoUsers: ApprovedUser[]): Promise => { let egaUsers: EgaUser[] = []; for await (const user of dacoUsers) { try { @@ -245,7 +256,7 @@ export const egaApiClient = async () => { * @param datasetAccessionId: DatasetAccessionId * @param limit number * @param offset number - * @returns EgaPermission[] + * @returns ZodResultAccumulator */ const getPermissionsForDataset = async ({ datasetAccessionId, @@ -275,15 +286,16 @@ export const egaApiClient = async () => { }; /** - * GET request to retrieve existing dataset permissions for a user + * GET request to retrieve existing dataset permissions for a user. + * One permission result is expected with userId and datasetId params, but response from EGA API comes as an array * @param userId string * @param datasetId DatasetAccessionId - * @returns EgaPermission[] + * @returns ZodResultAccumulator */ const getPermissionByDatasetAndUserId = async ( userId: string, datasetId: DatasetAccessionId, - ): Promise => { + ): Promise | undefined> => { try { const url = urlJoin(DACS, dacId, PERMISSIONS); const { data } = await apiAxiosClient.get(url, { @@ -295,19 +307,18 @@ export const egaApiClient = async () => { if (!data.length) { return undefined; } - const result = EgaPermission.safeParse(data[0]); - if (result.success) { - return result.data; - } + const result = safeParseArray(EgaPermission, data); + return result; } catch (err) { logger.error('Error retrieving permission for user'); + return undefined; } }; /** * POST request to create PermissionRequests for a user * @param requests PermissionRequest[] - * @returns EgaPermissionRequest[] + * @returns ZodResultAccumulator * @example * // returns [ * { @@ -338,12 +349,13 @@ export const egaApiClient = async () => { */ const createPermissionRequests = async ( requests: PermissionRequest[], - ): Promise => { + ): Promise | undefined> => { try { const { data } = await apiAxiosClient.post(REQUESTS, { requests, }); - return data; + const result = safeParseArray(EgaPermissionRequest, data); + return result; } catch (err) { logger.error('Create permissions request failed'); return undefined; diff --git a/src/jobs/ega/errors.ts b/src/jobs/ega/errors.ts index 21d6397..764ba76 100644 --- a/src/jobs/ega/errors.ts +++ b/src/jobs/ega/errors.ts @@ -19,6 +19,11 @@ import { AxiosError } from 'axios'; +/** + * Custom errors for Axios responses. + * Defines expected status and code values for error handling. + */ + export class NotFoundError extends AxiosError { constructor(message: string) { super(message); @@ -27,3 +32,12 @@ export class NotFoundError extends AxiosError { this.code = 'NOT_FOUND'; } } + +export class TooManyRequestsError extends AxiosError { + constructor(message: string) { + super(message); + this.name = 'TooManyRequests'; + this.status = 429; + this.code = 'TOO_MANY_REQUESTS'; + } +} diff --git a/src/jobs/ega/types.ts b/src/jobs/ega/types.ts index 7fa9876..2064aea 100644 --- a/src/jobs/ega/types.ts +++ b/src/jobs/ega/types.ts @@ -17,7 +17,9 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { z } from 'zod'; +import { z, ZodError, ZodTypeAny } from 'zod'; + +/** Common types */ // For YYYY-MM-DD Date strings (i.e. '2021-01-01') export const DateString = z.string().date(); @@ -27,11 +29,17 @@ export type DateString = z.infer; export const DateTime = z.string().datetime(); export type DateTime = z.infer; -// Enums +/** Enums & Literals */ + const DAC_STATUS_ENUM = ['accepted', 'pending', 'declined'] as const; export const DacStatus = z.enum(DAC_STATUS_ENUM); export type DacStatus = z.infer; +const IdpTokenType = z.literal('Bearer'); +type IdpTokenType = z.infer; + +/** Regexes */ + const DAC_ACCESSION_ID_REGEX = new RegExp(`^EGAC\\d{11}$`); export const DacAccessionId = z.string().regex(DAC_ACCESSION_ID_REGEX); export type DacAccessionId = z.infer; @@ -44,7 +52,20 @@ const USER_ACCESSION_ID_REGEX = new RegExp(`^EGAW\\d{11}$`); export const UserAccessionId = z.string().regex(USER_ACCESSION_ID_REGEX); export type UserAccessionId = z.infer; -// EGA Response Types +/** EGA Response Types */ + +export const IdpToken = z.object({ + access_token: z.string(), + scope: z.string(), + session_state: z.string(), + token_type: IdpTokenType, + refresh_token: z.string(), + refresh_expires_in: z.number(), + expires_in: z.number(), + 'not-before-policy': z.number(), +}); +export type IdpToken = z.infer; + export const Dac = z.object({ provisional_id: z.number(), accession_id: z.string(), @@ -106,13 +127,8 @@ export type ApprovePermissionResponse = z.infer; -// Axios -export type Success = { - success: true; - data: T; -}; +/** Request Data Types */ -// Request Data Types export type PermissionRequest = { username: string; dataset_accession_id: DatasetAccessionId; @@ -130,3 +146,38 @@ export type RevokePermission = { id: number; reason: string; }; + +/** Utility Functions */ + +export type ZodResultAccumulator = { success: T[]; failure: ZodError[] }; +/** + * Parses an array of Zod SafeParseReturnType results into success (successful parse) and failure (parsing error) + * @param acc ZodResultAccumulator + * @param item z.SafeParseReturnType + * @returns ZodResultAccumulator + */ +const resultReducer = (acc: ZodResultAccumulator, item: z.SafeParseReturnType) => { + if (item.success) { + acc.success.push(item.data); + } else { + acc.failure.push(item.error); + } + return acc; +}; + +/** + * Run Zod safeParse for Schema T on an array of items, and split results by SafeParseReturnType 'success' or 'error'. + * @params schema + * @params data unknown[] + * @returns { success: [], failure: [] } + */ +export const safeParseArray = ( + schema: T, + data: Array, +): ZodResultAccumulator> => + data + .map((i) => schema.safeParse(i)) + .reduce>>((acc, item) => resultReducer(acc, item), { + success: [], + failure: [], + }); diff --git a/src/jobs/ega/utils.ts b/src/jobs/ega/utils.ts index 35a6319..750bb05 100644 --- a/src/jobs/ega/utils.ts +++ b/src/jobs/ega/utils.ts @@ -18,12 +18,11 @@ */ import { uniqBy } from 'lodash'; -import { ZodError, ZodTypeAny, z } from 'zod'; import { UserDataFromApprovedApplicationsResult } from '../../domain/interface'; import { getUsersFromApprovedApps } from '../../domain/service/applications/search'; import { DatasetAccessionId, PermissionRequest } from './types'; -type ApprovedUser = { +export type ApprovedUser = { username: string; email: string; affiliation: string; @@ -85,36 +84,3 @@ const createPermissionRequest = ( }, }; }; - -export type ZodResultAccumulator = { success: T[]; failure: ZodError[] }; -/** - * Parses an array of Zod SafeParseReturnType results into success (successful parse) and failure (parsing error) - * @param acc ZodResultAccumulator - * @param item z.SafeParseReturnType - * @returns ZodResultAccumulator - */ -const resultReducer = (acc: ZodResultAccumulator, item: z.SafeParseReturnType) => { - if (item.success) { - acc.success.push(item.data); - } else { - acc.failure.push(item.error); - } - return acc; -}; - -/** - * Run Zod safeParse for Schema T on an array of items, and split results by SafeParseReturnType 'success' or 'error'. - * @params schema - * @params data unknown[] - * @returns { success: [], failure: [] } - */ -export const safeParseArray = ( - schema: T, - data: Array, -): ZodResultAccumulator> => - data - .map((i) => schema.safeParse(i)) - .reduce>>((acc, item) => resultReducer(acc, item), { - success: [], - failure: [], - }); From f62d34b8ac49e4f7f76009ba7dfc01fae95a69d8 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Wed, 25 Sep 2024 18:25:27 -0400 Subject: [PATCH 13/25] Add failure returns to ega client calls, restructure types --- src/jobs/ega/egaClient.ts | 141 +++++++++++------- src/jobs/ega/types/common.ts | 59 ++++++++ src/jobs/ega/types/index.ts | 0 src/jobs/ega/types/requests.ts | 38 +++++ src/jobs/ega/{types.ts => types/responses.ts} | 101 ++----------- src/jobs/ega/types/results.ts | 110 ++++++++++++++ src/jobs/ega/utils.ts | 25 +++- 7 files changed, 326 insertions(+), 148 deletions(-) create mode 100644 src/jobs/ega/types/common.ts create mode 100644 src/jobs/ega/types/index.ts create mode 100644 src/jobs/ega/types/requests.ts rename src/jobs/ega/{types.ts => types/responses.ts} (56%) create mode 100644 src/jobs/ega/types/results.ts diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts index 38a85ac..5ec9263 100644 --- a/src/jobs/ega/egaClient.ts +++ b/src/jobs/ega/egaClient.ts @@ -30,27 +30,31 @@ import { EGA_TOKEN_ENDPOINT, } from '../../utils/constants'; import { NotFoundError, TooManyRequestsError } from './errors'; +import { DacAccessionId, DatasetAccessionId } from './types/common'; +import { ApprovePermissionRequest, PermissionRequest, RevokePermission } from './types/requests'; import { - ApprovePermissionRequest, ApprovePermissionResponse, - DacAccessionId, Dataset, - DatasetAccessionId, EgaPermission, EgaPermissionRequest, EgaUser, IdpToken, - PermissionRequest, - RevokePermission, RevokePermissionResponse, +} from './types/responses'; +import { + Failure, + failure, + Result, safeParseArray, + success, ZodResultAccumulator, -} from './types'; -import { ApprovedUser } from './utils'; +} from './types/results'; +import { ApprovedUser, getErrorMessage } from './utils'; const { DACS, DATASETS, PERMISSIONS, REQUESTS, USERS } = EGA_API; +type ServerError = 'SERVER_ERROR'; -// initialize idp client +// initialize IDP client const initIdpClient = () => { const { ega: { authHost }, @@ -188,6 +192,7 @@ export const egaApiClient = async () => { }, ); + type GetDatasetsForDacFailure = ServerError; /** * GET request to retrieve all currently release datasets released for a DAC * @param dacId DacAccessionId @@ -195,61 +200,59 @@ export const egaApiClient = async () => { */ const getDatasetsForDac = async ( dacId: DacAccessionId, - ): Promise | []> => { + ): Promise | Failure> => { const url = urlJoin(DACS, dacId, DATASETS); try { const { data } = await apiAxiosClient.get(url); const result = safeParseArray(Dataset, data); return result; } catch (err) { + const errMessage = getErrorMessage(err, `Error retrieving datasets for DAC ${dacId}.`); logger.error(`Error retrieving datasets for DAC ${dacId}.`); - return []; + return failure('SERVER_ERROR', errMessage); } }; + type GetUserFailure = 'SERVER_ERROR' | 'NOT_FOUND' | 'INVALID_USER'; /** - * Retrieve EGA user data for each user on DACO approved list - * @param dacoUsers ApprovedUser[] - * @returns EGAUser[] + * Retrieve EGA user data for a DACO ApprovedUser + * @returns EGAUser * @example - * // returns [ + * // returns * { * id: 123, * username: boysue@example.com, * email: boysue@example.com, * accession_id: EGAW00000009999 * } - * ] * getUser('boysue@example.com') */ - const getUsers = async (dacoUsers: ApprovedUser[]): Promise => { - let egaUsers: EgaUser[] = []; - for await (const user of dacoUsers) { - try { - const { data } = await apiAxiosClient.get(urlJoin(USERS, user.email)); - const egaUser = EgaUser.safeParse(data); - if (egaUser.success) { - logger.info('Successfully parsed ', user.email, '. Adding to list.'); - egaUsers.push(egaUser.data); - } - } catch (err) { - if (err instanceof AxiosError) { - switch (err.code) { - case 'NOT_FOUND': - // TODO: add user to error report? - logger.error('User not found'); - break; - default: - logger.error('Axios error'); - } - } else { - logger.error('System error'); + const getUser = async (user: ApprovedUser): Promise> => { + const url = urlJoin(USERS, user.email); + try { + const { data } = await apiAxiosClient.get(url); + const egaUser = EgaUser.safeParse(data); + if (egaUser.success) { + return success(egaUser.data); + } + return failure('INVALID_USER', 'Failed to parse user response'); + } catch (err) { + if (err instanceof AxiosError) { + switch (err.code) { + case 'NOT_FOUND': + return failure('NOT_FOUND', 'User not found'); + default: + return failure('SERVER_ERROR', 'Axios error'); } + } else { + const errMessage = getErrorMessage(err, 'Get user request failed'); + logger.error('Get user request failed'); + return failure('SERVER_ERROR', errMessage); } } - return egaUsers; }; + type GetPermissionsForDatasetFailure = ServerError; /** * GET request for list of existing permissions for a dataset * Endpoint is paginated. @@ -266,7 +269,7 @@ export const egaApiClient = async () => { datasetAccessionId: DatasetAccessionId; limit: number; offset: number; - }): Promise | undefined> => { + }): Promise | Failure> => { const url = urlJoin(DACS, dacId, PERMISSIONS); try { const { data } = await apiAxiosClient.get(url, { @@ -280,11 +283,13 @@ export const egaApiClient = async () => { const result = safeParseArray(EgaPermission, data); return result; } catch (err) { - logger.error(err); - return undefined; + const errMessage = getErrorMessage(err, 'Get permissions for dataset request failed.'); + logger.error('Get permissions for dataset request failed.'); + return failure('SERVER_ERROR', errMessage); } }; + type GetPermissionsByDatasetAndUserIdFailure = ServerError; /** * GET request to retrieve existing dataset permissions for a user. * One permission result is expected with userId and datasetId params, but response from EGA API comes as an array @@ -295,7 +300,9 @@ export const egaApiClient = async () => { const getPermissionByDatasetAndUserId = async ( userId: string, datasetId: DatasetAccessionId, - ): Promise | undefined> => { + ): Promise< + ZodResultAccumulator | Failure + > => { try { const url = urlJoin(DACS, dacId, PERMISSIONS); const { data } = await apiAxiosClient.get(url, { @@ -304,17 +311,16 @@ export const egaApiClient = async () => { user_id: userId, }, }); - if (!data.length) { - return undefined; - } const result = safeParseArray(EgaPermission, data); return result; } catch (err) { + const errMessage = getErrorMessage(err, 'Error retrieving permission for user'); logger.error('Error retrieving permission for user'); - return undefined; + return failure('SERVER_ERROR', errMessage); } }; + type CreatePermissionRequestsFailure = ServerError; /** * POST request to create PermissionRequests for a user * @param requests PermissionRequest[] @@ -349,7 +355,9 @@ export const egaApiClient = async () => { */ const createPermissionRequests = async ( requests: PermissionRequest[], - ): Promise | undefined> => { + ): Promise< + ZodResultAccumulator | Failure + > => { try { const { data } = await apiAxiosClient.post(REQUESTS, { requests, @@ -357,11 +365,15 @@ export const egaApiClient = async () => { const result = safeParseArray(EgaPermissionRequest, data); return result; } catch (err) { + const errMessage = getErrorMessage(err, 'Create permissions request failed.'); logger.error('Create permissions request failed'); - return undefined; + return failure('SERVER_ERROR', errMessage); } }; + type ApprovedPermissionRequestsFailure = + | ServerError + | 'INVALID_APPROVE_PERMISSION_REQUESTS_RESPONSE'; /** * Approves permissions by permission id. * Endpoint accepts an array so multiple permissions can be approved in one request. @@ -378,17 +390,26 @@ export const egaApiClient = async () => { */ const approvePermissionRequests = async ( requests: ApprovePermissionRequest[], - ): Promise => { + ): Promise> => { try { const { data } = await apiAxiosClient.put(REQUESTS, { requests, }); - return data; + const result = ApprovePermissionResponse.safeParse(data); + if (result.success) { + return success(result.data); + } + return failure( + 'INVALID_APPROVE_PERMISSION_REQUESTS_RESPONSE', + 'Invalid response for approve permission requests.', + ); } catch (err) { + const errMessage = getErrorMessage(err, 'Approve permissions requests failed.'); logger.error('Create permissions request failed'); - return undefined; + return failure('SERVER_ERROR', errMessage); } }; + type RevokePermissionsFailure = ServerError | 'INVALID_REVOKE_PERMISSIONS_RESPONSE'; /** * Revokes permissions by permission id. @@ -406,13 +427,21 @@ export const egaApiClient = async () => { */ const revokePermissions = async ( requests: RevokePermission[], - ): Promise => { + ): Promise> => { try { const { data } = await apiAxiosClient.delete(PERMISSIONS, { data: requests }); - return data; + const result = RevokePermissionResponse.safeParse(data); + if (result.success) { + return success(result.data); + } + return failure( + 'INVALID_REVOKE_PERMISSIONS_RESPONSE', + 'Invalid response from revoke permissions request.', + ); } catch (err) { - logger.error('Create permissions request failed'); - return undefined; + const errMessage = getErrorMessage(err, 'Revoke permissions request failed'); + logger.error('Revoke permissions request failed'); + return failure('SERVER_ERROR', errMessage); } }; @@ -422,7 +451,7 @@ export const egaApiClient = async () => { getDatasetsForDac, getPermissionByDatasetAndUserId, getPermissionsForDataset, - getUsers, + getUser, revokePermissions, }; }; diff --git a/src/jobs/ega/types/common.ts b/src/jobs/ega/types/common.ts new file mode 100644 index 0000000..01f608d --- /dev/null +++ b/src/jobs/ega/types/common.ts @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { z } from 'zod'; + +/* ******************* * + Dates + * ******************* */ + +// For YYYY-MM-DD Date strings (i.e. '2021-01-01') +export const DateString = z.string().date(); +export type DateString = z.infer; + +// For ISO8601 Datetime strings (i.e. '2021-01-01T00:00:00.000Z') +export const DateTime = z.string().datetime(); +export type DateTime = z.infer; + +/* ******************* * + Enums & Literals + * ******************* */ + +const DAC_STATUS_ENUM = ['accepted', 'pending', 'declined'] as const; +export const DacStatus = z.enum(DAC_STATUS_ENUM); +export type DacStatus = z.infer; + +export const IdpTokenType = z.literal('Bearer'); +export type IdpTokenType = z.infer; + +/* ******************* * + Regexes + * ******************* */ + +const DAC_ACCESSION_ID_REGEX = new RegExp(`^EGAC\\d{11}$`); +export const DacAccessionId = z.string().regex(DAC_ACCESSION_ID_REGEX); +export type DacAccessionId = z.infer; + +const DATASET_ACCESSION_ID_REGEX = new RegExp(`^EGAD\\d{11}$`); +export const DatasetAccessionId = z.string().regex(DATASET_ACCESSION_ID_REGEX); +export type DatasetAccessionId = z.infer; + +const USER_ACCESSION_ID_REGEX = new RegExp(`^EGAW\\d{11}$`); +export const UserAccessionId = z.string().regex(USER_ACCESSION_ID_REGEX); +export type UserAccessionId = z.infer; diff --git a/src/jobs/ega/types/index.ts b/src/jobs/ega/types/index.ts new file mode 100644 index 0000000..e69de29 diff --git a/src/jobs/ega/types/requests.ts b/src/jobs/ega/types/requests.ts new file mode 100644 index 0000000..2fb2229 --- /dev/null +++ b/src/jobs/ega/types/requests.ts @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { DatasetAccessionId, DateTime } from './common'; + +export type PermissionRequest = { + username: string; + dataset_accession_id: DatasetAccessionId; + request_data: { + comment: string; + }; +}; + +export type ApprovePermissionRequest = { + request_id: number; + expires_at: DateTime; +}; + +export type RevokePermission = { + id: number; + reason: string; +}; diff --git a/src/jobs/ega/types.ts b/src/jobs/ega/types/responses.ts similarity index 56% rename from src/jobs/ega/types.ts rename to src/jobs/ega/types/responses.ts index 2064aea..e299730 100644 --- a/src/jobs/ega/types.ts +++ b/src/jobs/ega/types/responses.ts @@ -17,42 +17,16 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { z, ZodError, ZodTypeAny } from 'zod'; - -/** Common types */ - -// For YYYY-MM-DD Date strings (i.e. '2021-01-01') -export const DateString = z.string().date(); -export type DateString = z.infer; - -// For ISO8601 Datetime strings (i.e. '2021-01-01T00:00:00.000Z') -export const DateTime = z.string().datetime(); -export type DateTime = z.infer; - -/** Enums & Literals */ - -const DAC_STATUS_ENUM = ['accepted', 'pending', 'declined'] as const; -export const DacStatus = z.enum(DAC_STATUS_ENUM); -export type DacStatus = z.infer; - -const IdpTokenType = z.literal('Bearer'); -type IdpTokenType = z.infer; - -/** Regexes */ - -const DAC_ACCESSION_ID_REGEX = new RegExp(`^EGAC\\d{11}$`); -export const DacAccessionId = z.string().regex(DAC_ACCESSION_ID_REGEX); -export type DacAccessionId = z.infer; - -const DATASET_ACCESSION_ID_REGEX = new RegExp(`^EGAD\\d{11}$`); -export const DatasetAccessionId = z.string().regex(DATASET_ACCESSION_ID_REGEX); -export type DatasetAccessionId = z.infer; - -const USER_ACCESSION_ID_REGEX = new RegExp(`^EGAW\\d{11}$`); -export const UserAccessionId = z.string().regex(USER_ACCESSION_ID_REGEX); -export type UserAccessionId = z.infer; - -/** EGA Response Types */ +import { z } from 'zod'; +import { + DacAccessionId, + DacStatus, + DatasetAccessionId, + DateString, + DateTime, + IdpTokenType, + UserAccessionId, +} from './common'; export const IdpToken = z.object({ access_token: z.string(), @@ -126,58 +100,3 @@ export type ApprovePermissionResponse = z.infer; - -/** Request Data Types */ - -export type PermissionRequest = { - username: string; - dataset_accession_id: DatasetAccessionId; - request_data: { - comment: string; - }; -}; - -export type ApprovePermissionRequest = { - request_id: number; - expires_at: DateTime; -}; - -export type RevokePermission = { - id: number; - reason: string; -}; - -/** Utility Functions */ - -export type ZodResultAccumulator = { success: T[]; failure: ZodError[] }; -/** - * Parses an array of Zod SafeParseReturnType results into success (successful parse) and failure (parsing error) - * @param acc ZodResultAccumulator - * @param item z.SafeParseReturnType - * @returns ZodResultAccumulator - */ -const resultReducer = (acc: ZodResultAccumulator, item: z.SafeParseReturnType) => { - if (item.success) { - acc.success.push(item.data); - } else { - acc.failure.push(item.error); - } - return acc; -}; - -/** - * Run Zod safeParse for Schema T on an array of items, and split results by SafeParseReturnType 'success' or 'error'. - * @params schema - * @params data unknown[] - * @returns { success: [], failure: [] } - */ -export const safeParseArray = ( - schema: T, - data: Array, -): ZodResultAccumulator> => - data - .map((i) => schema.safeParse(i)) - .reduce>>((acc, item) => resultReducer(acc, item), { - success: [], - failure: [], - }); diff --git a/src/jobs/ega/types/results.ts b/src/jobs/ega/types/results.ts new file mode 100644 index 0000000..0dfb3ce --- /dev/null +++ b/src/jobs/ega/types/results.ts @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { z, ZodError, ZodTypeAny } from 'zod'; + +/* ******************* * + Success and Failure types + * ******************* */ + +export type Success = { status: 'SUCCESS'; data: T }; +export type Failure = { + status: FailureStatus; + message: string; + data: T; +}; + +/** + * Represents a response that on success will include data of type T, + * otherwise a message will be returned in place of the data explaining the failure with optional fallback data. + * The failure object has data type of void by default. + */ +export type Result = + | Success + | Failure; +/** + * Determines if the Result is a Success type by its status + * and returns the type predicate so TS can infer the Result as a Success + * @param result + * @returns {boolean} Whether the Result was a Success or not + */ + +/* ******************* * + Convenience methods + * ******************* */ + +export function isSuccess( + result: Result, +): result is Success { + return result.status === 'SUCCESS'; +} + +/** + * Create a successful response for a Result or Either type, with data of the success type + * @param {T} data + * @returns {Success} `{status: 'SUCCESS', data}` + */ +export const success = (data: T): Success => ({ status: 'SUCCESS', data }); + +/** + * Create a response indicating a failure with a status naming the reason and message describing the failure. + * @param {string} message + * @returns {Failure} `{status: string, message: string, data: undefined}` + */ +export const failure = ( + status: FailureStatus, + message: string, +): Failure => ({ + status, + message, + data: undefined, +}); + +export type ZodResultAccumulator = { success: T[]; failure: ZodError[] }; +/** + * Parses an array of Zod SafeParseReturnType results into success (successful parse) and failure (parsing error) + * @param acc ZodResultAccumulator + * @param item z.SafeParseReturnType + * @returns ZodResultAccumulator + */ +const resultReducer = (acc: ZodResultAccumulator, item: z.SafeParseReturnType) => { + if (item.success) { + acc.success.push(item.data); + } else { + acc.failure.push(item.error); + } + return acc; +}; + +/** + * Run Zod safeParse for Schema T on an array of items, and split results by SafeParseReturnType 'success' or 'error'. + * @params schema + * @params data unknown[] + * @returns { success: [], failure: [] } + */ +export const safeParseArray = ( + schema: T, + data: Array, +): ZodResultAccumulator> => + data + .map((i) => schema.safeParse(i)) + .reduce>>((acc, item) => resultReducer(acc, item), { + success: [], + failure: [], + }); diff --git a/src/jobs/ega/utils.ts b/src/jobs/ega/utils.ts index 750bb05..e8cf775 100644 --- a/src/jobs/ega/utils.ts +++ b/src/jobs/ega/utils.ts @@ -20,7 +20,8 @@ import { uniqBy } from 'lodash'; import { UserDataFromApprovedApplicationsResult } from '../../domain/interface'; import { getUsersFromApprovedApps } from '../../domain/service/applications/search'; -import { DatasetAccessionId, PermissionRequest } from './types'; +import { DatasetAccessionId } from './types/common'; +import { PermissionRequest, RevokePermission } from './types/requests'; export type ApprovedUser = { username: string; @@ -84,3 +85,25 @@ const createPermissionRequest = ( }, }; }; + +/** + * Create revoke permission request object for DELETE /requests + * @param permissionId + * @returns RevokePermissionRequest + */ +const createRevokePermissionRequest = (permissionId: number): RevokePermission => { + return { + id: permissionId, + reason: 'ICGC DAC access has expired.', + }; +}; + +/** + * Checks if error arg is of type Error, and returns err.message if so; otherwise returns defaultMessage arg + * Used in catch block of try/catch, where type of error in catch is unknown + * @param error unknown + * @param defaultMessage string + * @returns string + */ +export const getErrorMessage = (error: unknown, defaultMessage: string): string => + error instanceof Error ? error.message : defaultMessage; From 14d35f4457a186ce415d02bff8455dd9f0638771 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Wed, 25 Sep 2024 18:29:02 -0400 Subject: [PATCH 14/25] remove empty file --- src/jobs/ega/types/index.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/jobs/ega/types/index.ts diff --git a/src/jobs/ega/types/index.ts b/src/jobs/ega/types/index.ts deleted file mode 100644 index e69de29..0000000 From 12bb8904715f8d6afcf7d7400ca817616427ff7f Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Thu, 26 Sep 2024 08:29:17 -0400 Subject: [PATCH 15/25] move failure types to separate file, add appId to approved users data --- src/jobs/ega/egaClient.ts | 17 +++++++---------- src/jobs/ega/types/results.ts | 19 +++++++++++++++++-- src/jobs/ega/utils.ts | 9 +++------ 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts index 5ec9263..be8eaea 100644 --- a/src/jobs/ega/egaClient.ts +++ b/src/jobs/ega/egaClient.ts @@ -42,9 +42,16 @@ import { RevokePermissionResponse, } from './types/responses'; import { + ApprovedPermissionRequestsFailure, + CreatePermissionRequestsFailure, Failure, failure, + GetDatasetsForDacFailure, + GetPermissionsByDatasetAndUserIdFailure, + GetPermissionsForDatasetFailure, + GetUserFailure, Result, + RevokePermissionsFailure, safeParseArray, success, ZodResultAccumulator, @@ -52,7 +59,6 @@ import { import { ApprovedUser, getErrorMessage } from './utils'; const { DACS, DATASETS, PERMISSIONS, REQUESTS, USERS } = EGA_API; -type ServerError = 'SERVER_ERROR'; // initialize IDP client const initIdpClient = () => { @@ -192,7 +198,6 @@ export const egaApiClient = async () => { }, ); - type GetDatasetsForDacFailure = ServerError; /** * GET request to retrieve all currently release datasets released for a DAC * @param dacId DacAccessionId @@ -213,7 +218,6 @@ export const egaApiClient = async () => { } }; - type GetUserFailure = 'SERVER_ERROR' | 'NOT_FOUND' | 'INVALID_USER'; /** * Retrieve EGA user data for a DACO ApprovedUser * @returns EGAUser @@ -252,7 +256,6 @@ export const egaApiClient = async () => { } }; - type GetPermissionsForDatasetFailure = ServerError; /** * GET request for list of existing permissions for a dataset * Endpoint is paginated. @@ -289,7 +292,6 @@ export const egaApiClient = async () => { } }; - type GetPermissionsByDatasetAndUserIdFailure = ServerError; /** * GET request to retrieve existing dataset permissions for a user. * One permission result is expected with userId and datasetId params, but response from EGA API comes as an array @@ -320,7 +322,6 @@ export const egaApiClient = async () => { } }; - type CreatePermissionRequestsFailure = ServerError; /** * POST request to create PermissionRequests for a user * @param requests PermissionRequest[] @@ -371,9 +372,6 @@ export const egaApiClient = async () => { } }; - type ApprovedPermissionRequestsFailure = - | ServerError - | 'INVALID_APPROVE_PERMISSION_REQUESTS_RESPONSE'; /** * Approves permissions by permission id. * Endpoint accepts an array so multiple permissions can be approved in one request. @@ -409,7 +407,6 @@ export const egaApiClient = async () => { return failure('SERVER_ERROR', errMessage); } }; - type RevokePermissionsFailure = ServerError | 'INVALID_REVOKE_PERMISSIONS_RESPONSE'; /** * Revokes permissions by permission id. diff --git a/src/jobs/ega/types/results.ts b/src/jobs/ega/types/results.ts index 0dfb3ce..4c98b04 100644 --- a/src/jobs/ega/types/results.ts +++ b/src/jobs/ega/types/results.ts @@ -20,7 +20,7 @@ import { z, ZodError, ZodTypeAny } from 'zod'; /* ******************* * - Success and Failure types + Success and Failure types * ******************* */ export type Success = { status: 'SUCCESS'; data: T }; @@ -46,7 +46,7 @@ export type Result = */ /* ******************* * - Convenience methods + Convenience methods * ******************* */ export function isSuccess( @@ -108,3 +108,18 @@ export const safeParseArray = ( success: [], failure: [], }); + +/* ******************* * + Failure types + * ******************* */ + +export type ServerError = 'SERVER_ERROR'; +export type GetDatasetsForDacFailure = ServerError; +export type GetPermissionsForDatasetFailure = ServerError; +export type GetPermissionsByDatasetAndUserIdFailure = ServerError; +export type CreatePermissionRequestsFailure = ServerError; +export type ApprovedPermissionRequestsFailure = + | ServerError + | 'INVALID_APPROVE_PERMISSION_REQUESTS_RESPONSE'; +export type RevokePermissionsFailure = ServerError | 'INVALID_REVOKE_PERMISSIONS_RESPONSE'; +export type GetUserFailure = ServerError | 'NOT_FOUND' | 'INVALID_USER'; diff --git a/src/jobs/ega/utils.ts b/src/jobs/ega/utils.ts index e8cf775..f96d671 100644 --- a/src/jobs/ega/utils.ts +++ b/src/jobs/ega/utils.ts @@ -24,10 +24,9 @@ import { DatasetAccessionId } from './types/common'; import { PermissionRequest, RevokePermission } from './types/requests'; export type ApprovedUser = { - username: string; email: string; - affiliation: string; appExpiry: Date; + appId: string; }; /** @@ -40,16 +39,14 @@ const parseApprovedUsersForApplication = ( ): ApprovedUser[] => { const applicantInfo = applicationData.applicant.info; const applicant = { - username: applicantInfo.displayName, email: applicantInfo.institutionEmail, - affiliation: applicantInfo.primaryAffiliation, appExpiry: applicationData.expiresAtUtc, + appId: applicationData.appId, }; const collabs = (applicationData.collaborators.list || []).map((collab) => ({ - username: collab.info.displayName, email: collab.info.institutionEmail, - affiliation: collab.info.primaryAffiliation, appExpiry: applicationData.expiresAtUtc, + appId: applicationData.appId, })); return [applicant, ...collabs].flat(); From e995c7912f033d05eb4f2be195d758a1e6f085a1 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Thu, 26 Sep 2024 14:44:14 -0400 Subject: [PATCH 16/25] modify list response types --- src/jobs/ega/egaClient.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts index be8eaea..18b68d2 100644 --- a/src/jobs/ega/egaClient.ts +++ b/src/jobs/ega/egaClient.ts @@ -44,7 +44,6 @@ import { import { ApprovedPermissionRequestsFailure, CreatePermissionRequestsFailure, - Failure, failure, GetDatasetsForDacFailure, GetPermissionsByDatasetAndUserIdFailure, @@ -205,12 +204,12 @@ export const egaApiClient = async () => { */ const getDatasetsForDac = async ( dacId: DacAccessionId, - ): Promise | Failure> => { + ): Promise, GetDatasetsForDacFailure>> => { const url = urlJoin(DACS, dacId, DATASETS); try { const { data } = await apiAxiosClient.get(url); const result = safeParseArray(Dataset, data); - return result; + return success(result); } catch (err) { const errMessage = getErrorMessage(err, `Error retrieving datasets for DAC ${dacId}.`); logger.error(`Error retrieving datasets for DAC ${dacId}.`); @@ -272,7 +271,7 @@ export const egaApiClient = async () => { datasetAccessionId: DatasetAccessionId; limit: number; offset: number; - }): Promise | Failure> => { + }): Promise, GetPermissionsForDatasetFailure>> => { const url = urlJoin(DACS, dacId, PERMISSIONS); try { const { data } = await apiAxiosClient.get(url, { @@ -284,7 +283,7 @@ export const egaApiClient = async () => { }); const result = safeParseArray(EgaPermission, data); - return result; + return success(result); } catch (err) { const errMessage = getErrorMessage(err, 'Get permissions for dataset request failed.'); logger.error('Get permissions for dataset request failed.'); @@ -300,10 +299,10 @@ export const egaApiClient = async () => { * @returns ZodResultAccumulator */ const getPermissionByDatasetAndUserId = async ( - userId: string, + userId: number, datasetId: DatasetAccessionId, ): Promise< - ZodResultAccumulator | Failure + Result, GetPermissionsByDatasetAndUserIdFailure> > => { try { const url = urlJoin(DACS, dacId, PERMISSIONS); @@ -314,7 +313,7 @@ export const egaApiClient = async () => { }, }); const result = safeParseArray(EgaPermission, data); - return result; + return success(result); } catch (err) { const errMessage = getErrorMessage(err, 'Error retrieving permission for user'); logger.error('Error retrieving permission for user'); @@ -357,14 +356,14 @@ export const egaApiClient = async () => { const createPermissionRequests = async ( requests: PermissionRequest[], ): Promise< - ZodResultAccumulator | Failure + Result, CreatePermissionRequestsFailure> > => { try { const { data } = await apiAxiosClient.post(REQUESTS, { requests, }); const result = safeParseArray(EgaPermissionRequest, data); - return result; + return success(result); } catch (err) { const errMessage = getErrorMessage(err, 'Create permissions request failed.'); logger.error('Create permissions request failed'); From 965f0bbfaca56cbf1c2d4a8cede8956c9f9bb1d2 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Mon, 30 Sep 2024 10:21:24 -0400 Subject: [PATCH 17/25] Create ega permissions main job function, add happy path flow --- src/jobs/ega/types/common.ts | 1 + src/jobs/ega/types/responses.ts | 5 + src/jobs/ega/utils.ts | 40 +++- src/jobs/egaPermissionsReconciliation.ts | 225 +++++++++++++++++++++++ 4 files changed, 267 insertions(+), 4 deletions(-) create mode 100644 src/jobs/egaPermissionsReconciliation.ts diff --git a/src/jobs/ega/types/common.ts b/src/jobs/ega/types/common.ts index 01f608d..1900491 100644 --- a/src/jobs/ega/types/common.ts +++ b/src/jobs/ega/types/common.ts @@ -28,6 +28,7 @@ export const DateString = z.string().date(); export type DateString = z.infer; // For ISO8601 Datetime strings (i.e. '2021-01-01T00:00:00.000Z') +// Note: for safeParse to allow the +00:00, as in '2024-01-31T16:25:13.725724+00:00', would need .datetime({ offset: true }) export const DateTime = z.string().datetime(); export type DateTime = z.infer; diff --git a/src/jobs/ega/types/responses.ts b/src/jobs/ega/types/responses.ts index e299730..13dc2eb 100644 --- a/src/jobs/ega/types/responses.ts +++ b/src/jobs/ega/types/responses.ts @@ -100,3 +100,8 @@ export type ApprovePermissionResponse = z.infer; + +export const EgaDacoUser = EgaUser.merge(z.object({ appExpiry: DateTime, appId: z.string() })); +export type EgaDacoUser = z.infer; + +export type EgaDacoUserMap = Record; diff --git a/src/jobs/ega/utils.ts b/src/jobs/ega/utils.ts index f96d671..9e0335d 100644 --- a/src/jobs/ega/utils.ts +++ b/src/jobs/ega/utils.ts @@ -20,8 +20,9 @@ import { uniqBy } from 'lodash'; import { UserDataFromApprovedApplicationsResult } from '../../domain/interface'; import { getUsersFromApprovedApps } from '../../domain/service/applications/search'; -import { DatasetAccessionId } from './types/common'; -import { PermissionRequest, RevokePermission } from './types/requests'; +import { DatasetAccessionId, DateTime } from './types/common'; +import { ApprovePermissionRequest, PermissionRequest, RevokePermission } from './types/requests'; +import { ApprovePermissionResponse, RevokePermissionResponse } from './types/responses'; export type ApprovedUser = { email: string; @@ -70,7 +71,7 @@ export const getApprovedUsers = async () => { * @param dataset_accession_id * @returns PermissionRequest */ -const createPermissionRequest = ( +export const createPermissionRequest = ( username: string, datasetAccessionId: DatasetAccessionId, ): PermissionRequest => { @@ -83,12 +84,27 @@ const createPermissionRequest = ( }; }; +/** + * Create EGA permission approval request object for PUT /requests + * Expiry date of approved DACO application is used for the permission expires_at value + * @param permissionRequestId number + * @param appExpiry Date + * @returns ApprovePermissionRequest + */ +export const createPermissionApprovalRequest = ( + permissionRequestId: number, + appExpiry: DateTime, +): ApprovePermissionRequest => ({ + request_id: permissionRequestId, + expires_at: appExpiry, +}); + /** * Create revoke permission request object for DELETE /requests * @param permissionId * @returns RevokePermissionRequest */ -const createRevokePermissionRequest = (permissionId: number): RevokePermission => { +export const createRevokePermissionRequest = (permissionId: number): RevokePermission => { return { id: permissionId, reason: 'ICGC DAC access has expired.', @@ -104,3 +120,19 @@ const createRevokePermissionRequest = (permissionId: number): RevokePermission = */ export const getErrorMessage = (error: unknown, defaultMessage: string): string => error instanceof Error ? error.message : defaultMessage; + +/** + * + */ +export const verifyPermissionApprovals = ( + numRequests: number, + approvalResponse: ApprovePermissionResponse, +): boolean => numRequests === approvalResponse.num_granted; + +/** + * + */ +export const verifyPermissionRevocations = ( + numRequests: number, + revokeResponse: RevokePermissionResponse, +): boolean => numRequests === revokeResponse.num_revoked; diff --git a/src/jobs/egaPermissionsReconciliation.ts b/src/jobs/egaPermissionsReconciliation.ts new file mode 100644 index 0000000..575fefd --- /dev/null +++ b/src/jobs/egaPermissionsReconciliation.ts @@ -0,0 +1,225 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { getAppConfig } from '../config'; +import logger, { buildMessage } from '../logger'; +import { egaApiClient, EgaClient } from './ega/egaClient'; +import { DatasetAccessionId } from './ega/types/common'; +import { RevokePermission } from './ega/types/requests'; +import { Dataset, EgaDacoUserMap } from './ega/types/responses'; +import { isSuccess } from './ega/types/results'; +import { + ApprovedUser, + createPermissionApprovalRequest, + createPermissionRequest, + createRevokePermissionRequest, + getApprovedUsers, + verifyPermissionApprovals, +} from './ega/utils'; + +const JOB_NAME = 'RECONCILE_EGA_PERMISSIONS'; + +/** + * Retrieve EGA user data for each user on DACO approved list + * @param client EgaClient + * @param dacoUsers ApprovedUser[] + * @returns EgaDacoUserMap + * @example + * // returns { + * boysue@example.com: { + * id: 123, + * username: boysue@example.com, + * email: boysue@example.com, + * accession_id: EGAW00000009999, + * appExpiry: '2024-10-01T14:06:41.485Z', + * appId: 'DACO-1' + * } + * ... + * } + * getUsers(client, approvedUsersList) + */ +const getUsers = async ( + client: EgaClient, + approvedUsers: ApprovedUser[], +): Promise => { + let egaUsers: EgaDacoUserMap = {}; + for await (const user of approvedUsers) { + try { + const egaUser = await client.getUser(user); + if (egaUser.status === 'SUCCESS') { + const { data } = egaUser; + const egaDacoUser = { + ...data, + appExpiry: user.appExpiry.toDateString(), + appId: user.appId, + }; + egaUsers[data.username] = egaDacoUser; + } + } catch (err) { + logger.error(err); + } + } + return egaUsers; +}; + +const processPermissionsForApprovedUsers = async ( + egaClient: EgaClient, + egaUsers: EgaDacoUserMap, + datasets: Dataset[], +) => { + const userList = Object.values(egaUsers); + + for await (const approvedUser of userList) { + const permissionRequests = []; + for await (const dataset of datasets) { + // check for existing permission + const existingPermission = await egaClient.getPermissionByDatasetAndUserId( + approvedUser.id, + dataset.accession_id, + ); + if (isSuccess(existingPermission)) { + if (existingPermission.data.success.length === 0) { + // create permission request, add to requestList + const permissionRequest = createPermissionRequest( + approvedUser.username, + dataset.accession_id, + ); + permissionRequests.push(permissionRequest); + } + } + } + if (permissionRequests.length) { + // POST all requests + const createRequestsResponse = await egaClient.createPermissionRequests(permissionRequests); + if (!isSuccess(createRequestsResponse)) { + throw new Error('Failed to create permissions requests'); + } + // create approval requests objs + send all + const approvalRequests = createRequestsResponse.data.success.map((request) => + createPermissionApprovalRequest(request.request_id, approvedUser.appExpiry), + ); + const approvePermissionRequestsResponse = await egaClient.approvePermissionRequests( + approvalRequests, + ); + if (isSuccess(approvePermissionRequestsResponse)) { + verifyPermissionApprovals(approvalRequests.length, approvePermissionRequestsResponse.data); + } + } + } + logger.info('Completed processing permissions for all DACO approved users.'); +}; + +const DEFAULT_OFFSET = 50; +const DEFAULT_LIMIT = 50; + +/** + * Paginates through all permissions for a dataset and revokes permissions for users not found in approvedUsers + * @param client EgaClient + * @param dataset_accession_id DatasetAccessionId + * @param approvedUsers EgaDacoUserMap + * @returns void + */ +export const processPermissionsForDataset = async ( + client: EgaClient, + datasetAccessionId: DatasetAccessionId, + approvedUsers: EgaDacoUserMap, +): Promise => { + let permissionsToRevoke: RevokePermission[] = []; + let offset = 0; + let limit = DEFAULT_LIMIT; + let paging = true; + + // loop will stop once result length from GET is less than limit + while (paging) { + const permissions = await client.getPermissionsForDataset({ + datasetAccessionId, + limit, + offset, + }); + if (isSuccess(permissions)) { + const { success: permissionsSuccesses, failure: permissionsFailures } = permissions.data; + permissionsSuccesses.map((permission) => { + // check if permission username is found in approvedUsers + const hasAccess = approvedUsers[permission.username]; + if (!hasAccess) { + const revokeRequest = createRevokePermissionRequest(permission.permission_id); + permissionsToRevoke.push(revokeRequest); + } + }); + offset = offset + DEFAULT_OFFSET; + const totalResults = permissionsFailures.length + permissionsSuccesses.length; + paging = totalResults >= limit; + } + } + if (permissionsToRevoke.length) { + const revokeResponse = await client.revokePermissions(permissionsToRevoke); + if (isSuccess(revokeResponse)) { + logger.info( + `Successfully revoked ${revokeResponse.data.num_revoked} of total ${permissionsToRevoke.length} permissions for DATASET ${datasetAccessionId}.`, + ); + } else { + logger.error( + `There was an error revoking permissions for DATASET ${datasetAccessionId} - ${revokeResponse.message}.`, + ); + } + } else { + logger.info(`There are no permissions to revoke for DATASET ${datasetAccessionId}.`); + } +}; + +/** + * Steps: + * 1) Retrieve approved users list from dac db + * 2) Retrieve datasets for DAC + * 3) Retrieve corresponding list of users from EGA API + * 4) Create permissions, on each dataset, for each user on the DACO approved list, if no existing permission is found + * 5) Process existing permissions for each dataset + revoke those which belong to users not on the DACO approved list + */ +export default async function () { + // retrieve approved users list from daco system + const dacoUsers = await getApprovedUsers(); + // initialize EGA Axios client + const egaClient = await egaApiClient(); + + // retrieve all datasets for ICGC DAC + const { + ega: { dacId }, + } = getAppConfig(); + const datasets = await egaClient.getDatasetsForDac(dacId); + + // get datasets failed completely + if (!isSuccess(datasets)) { + // TODO: retry here? + throw new Error('Failed to fetch datasets'); + } + // retrieve corresponding users in EGA system + const egaUsers = await getUsers(egaClient, dacoUsers); + const datasetsRetrieved = datasets.data.success; + // check DACO approved users have expected EGA permissions for each dataset + await processPermissionsForApprovedUsers(egaClient, egaUsers, datasetsRetrieved); + + // can add a return value to these process functions if needed, i.e. BatchJobReport + + // Check existing permissions per dataset + revoke if needed + for await (const dataset of datasetsRetrieved) { + await processPermissionsForDataset(egaClient, dataset.accession_id, egaUsers); + } + + logger.info(buildMessage(JOB_NAME, 'Completed.')); +} From e866b2bad383e95d73f257f6fe043e35375863cf Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Mon, 30 Sep 2024 10:26:53 -0400 Subject: [PATCH 18/25] move safeParseArray to own file, update imports --- src/jobs/ega/egaClient.ts | 3 +- src/jobs/ega/types/results.ts | 35 ---------------- src/jobs/ega/types/zodSafeParseArray.ts | 53 +++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 37 deletions(-) create mode 100644 src/jobs/ega/types/zodSafeParseArray.ts diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts index 18b68d2..56fbbee 100644 --- a/src/jobs/ega/egaClient.ts +++ b/src/jobs/ega/egaClient.ts @@ -51,10 +51,9 @@ import { GetUserFailure, Result, RevokePermissionsFailure, - safeParseArray, success, - ZodResultAccumulator, } from './types/results'; +import { safeParseArray, ZodResultAccumulator } from './types/zodSafeParseArray'; import { ApprovedUser, getErrorMessage } from './utils'; const { DACS, DATASETS, PERMISSIONS, REQUESTS, USERS } = EGA_API; diff --git a/src/jobs/ega/types/results.ts b/src/jobs/ega/types/results.ts index 4c98b04..9ba2b55 100644 --- a/src/jobs/ega/types/results.ts +++ b/src/jobs/ega/types/results.ts @@ -17,8 +17,6 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import { z, ZodError, ZodTypeAny } from 'zod'; - /* ******************* * Success and Failure types * ******************* */ @@ -76,39 +74,6 @@ export const failure = ( data: undefined, }); -export type ZodResultAccumulator = { success: T[]; failure: ZodError[] }; -/** - * Parses an array of Zod SafeParseReturnType results into success (successful parse) and failure (parsing error) - * @param acc ZodResultAccumulator - * @param item z.SafeParseReturnType - * @returns ZodResultAccumulator - */ -const resultReducer = (acc: ZodResultAccumulator, item: z.SafeParseReturnType) => { - if (item.success) { - acc.success.push(item.data); - } else { - acc.failure.push(item.error); - } - return acc; -}; - -/** - * Run Zod safeParse for Schema T on an array of items, and split results by SafeParseReturnType 'success' or 'error'. - * @params schema - * @params data unknown[] - * @returns { success: [], failure: [] } - */ -export const safeParseArray = ( - schema: T, - data: Array, -): ZodResultAccumulator> => - data - .map((i) => schema.safeParse(i)) - .reduce>>((acc, item) => resultReducer(acc, item), { - success: [], - failure: [], - }); - /* ******************* * Failure types * ******************* */ diff --git a/src/jobs/ega/types/zodSafeParseArray.ts b/src/jobs/ega/types/zodSafeParseArray.ts new file mode 100644 index 0000000..8151d05 --- /dev/null +++ b/src/jobs/ega/types/zodSafeParseArray.ts @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { z, ZodError, ZodTypeAny } from 'zod'; + +export type ZodResultAccumulator = { success: T[]; failure: ZodError[] }; +/** + * Parses an array of Zod SafeParseReturnType results into success (successful parse) and failure (parsing error) + * @param acc ZodResultAccumulator + * @param item z.SafeParseReturnType + * @returns ZodResultAccumulator + */ +const resultReducer = (acc: ZodResultAccumulator, item: z.SafeParseReturnType) => { + if (item.success) { + acc.success.push(item.data); + } else { + acc.failure.push(item.error); + } + return acc; +}; + +/** + * Run Zod safeParse for Schema T on an array of items, and split results by SafeParseReturnType 'success' or 'error'. + * @params schema + * @params data unknown[] + * @returns { success: [], failure: [] } + */ +export const safeParseArray = ( + schema: T, + data: Array, +): ZodResultAccumulator> => + data + .map((i) => schema.safeParse(i)) + .reduce>>((acc, item) => resultReducer(acc, item), { + success: [], + failure: [], + }); From ef484437b9ec81b8044c13e8997644e54b0226f7 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Wed, 2 Oct 2024 22:10:23 -0400 Subject: [PATCH 19/25] WIP - fix for pagination when checking permissions --- src/jobs/ega/egaClient.ts | 85 ++++++----- src/jobs/ega/errors.ts | 22 ++- src/jobs/ega/types/common.ts | 2 +- src/jobs/ega/types/responses.ts | 10 +- src/jobs/ega/types/results.ts | 5 +- src/jobs/ega/utils.ts | 14 +- src/jobs/egaPermissionsReconciliation.ts | 175 +++++++++++++++++------ 7 files changed, 214 insertions(+), 99 deletions(-) diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts index 56fbbee..4f998cf 100644 --- a/src/jobs/ega/egaClient.ts +++ b/src/jobs/ega/egaClient.ts @@ -29,7 +29,7 @@ import { EGA_REALMS_PATH, EGA_TOKEN_ENDPOINT, } from '../../utils/constants'; -import { NotFoundError, TooManyRequestsError } from './errors'; +import { BadRequestError, NotFoundError, ServerError, TooManyRequestsError } from './errors'; import { DacAccessionId, DatasetAccessionId } from './types/common'; import { ApprovePermissionRequest, PermissionRequest, RevokePermission } from './types/requests'; import { @@ -166,30 +166,33 @@ export const egaApiClient = async () => { (response) => response, async (error) => { if (error instanceof AxiosError) { - if (error.response && error.response.status === 401) { - logger.info('Access expired, attempting refresh'); - // Access token has expired, refresh it - try { - const newAccessToken = await refreshAccessToken(token); - // Update the request headers with the new access token - const headers = new AxiosHeaders(error.config?.headers); - headers.setAuthorization(`Bearer ${newAccessToken.access_token}`); - error.config = { - ...error.config, - headers, - }; - // Retry the original request - return apiAxiosClient(error.config); - } catch (refreshError) { - // Handle token refresh error - throw refreshError; - } - } - if (error.status === 404) { - throw new NotFoundError(error.message); - } - if (error.status === 429) { - throw new TooManyRequestsError(error.message); + switch (error.status) { + case 401: + logger.info('Access expired, attempting refresh'); + // Access token has expired, refresh it + try { + const newAccessToken = await refreshAccessToken(token); + // Update the request headers with the new access token + const headers = new AxiosHeaders(error.config?.headers); + headers.setAuthorization(`Bearer ${newAccessToken.access_token}`); + error.config = { + ...error.config, + headers, + }; + // Retry the original request + return apiAxiosClient(error.config); + } catch (refreshError) { + // Handle token refresh error + throw refreshError; + } + case 400: + return new BadRequestError(error.message); + case 404: + throw new NotFoundError(error.message); + case 429: + throw new TooManyRequestsError(error.message); + default: + throw new ServerError('Unexpected Axios Error'); } } return new Response('Server error', { status: 500 }); @@ -232,8 +235,8 @@ export const egaApiClient = async () => { const getUser = async (user: ApprovedUser): Promise> => { const url = urlJoin(USERS, user.email); try { - const { data } = await apiAxiosClient.get(url); - const egaUser = EgaUser.safeParse(data); + const response = await apiAxiosClient.get(url); + const egaUser = EgaUser.safeParse(response.data); if (egaUser.success) { return success(egaUser.data); } @@ -280,7 +283,6 @@ export const egaApiClient = async () => { offset, }, }); - const result = safeParseArray(EgaPermission, data); return success(result); } catch (err) { @@ -358,9 +360,7 @@ export const egaApiClient = async () => { Result, CreatePermissionRequestsFailure> > => { try { - const { data } = await apiAxiosClient.post(REQUESTS, { - requests, - }); + const { data } = await apiAxiosClient.post(REQUESTS, requests); const result = safeParseArray(EgaPermissionRequest, data); return success(result); } catch (err) { @@ -388,16 +388,14 @@ export const egaApiClient = async () => { requests: ApprovePermissionRequest[], ): Promise> => { try { - const { data } = await apiAxiosClient.put(REQUESTS, { - requests, - }); + const { data } = await apiAxiosClient.put(REQUESTS, requests); const result = ApprovePermissionResponse.safeParse(data); if (result.success) { return success(result.data); } return failure( 'INVALID_APPROVE_PERMISSION_REQUESTS_RESPONSE', - 'Invalid response for approve permission requests.', + `Invalid response from approve permission requests: ${result.error}`, ); } catch (err) { const errMessage = getErrorMessage(err, 'Approve permissions requests failed.'); @@ -424,16 +422,27 @@ export const egaApiClient = async () => { requests: RevokePermission[], ): Promise> => { try { - const { data } = await apiAxiosClient.delete(PERMISSIONS, { data: requests }); - const result = RevokePermissionResponse.safeParse(data); + const response = await apiAxiosClient.delete(PERMISSIONS, { data: requests }); + if (response.status === 400) { + throw new BadRequestError('Permission not found.'); + } + const result = RevokePermissionResponse.safeParse(response.data); if (result.success) { return success(result.data); } return failure( 'INVALID_REVOKE_PERMISSIONS_RESPONSE', - 'Invalid response from revoke permissions request.', + `Invalid response from revoke permissions request: ${result.error}`, ); } catch (err) { + if (err instanceof AxiosError) { + switch (err.code) { + case 'BAD_REQUEST': + return failure('PERMISSION_DOES_NOT_EXIST', 'Permission not found.'); + default: + return failure('SERVER_ERROR', 'Axios error'); + } + } const errMessage = getErrorMessage(err, 'Revoke permissions request failed'); logger.error('Revoke permissions request failed'); return failure('SERVER_ERROR', errMessage); diff --git a/src/jobs/ega/errors.ts b/src/jobs/ega/errors.ts index 764ba76..638f395 100644 --- a/src/jobs/ega/errors.ts +++ b/src/jobs/ega/errors.ts @@ -27,7 +27,7 @@ import { AxiosError } from 'axios'; export class NotFoundError extends AxiosError { constructor(message: string) { super(message); - this.name = 'NotFound'; + this.name = 'Not Found'; this.status = 404; this.code = 'NOT_FOUND'; } @@ -36,8 +36,26 @@ export class NotFoundError extends AxiosError { export class TooManyRequestsError extends AxiosError { constructor(message: string) { super(message); - this.name = 'TooManyRequests'; + this.name = 'Too Many Requests'; this.status = 429; this.code = 'TOO_MANY_REQUESTS'; } } + +export class BadRequestError extends AxiosError { + constructor(message: string) { + super(message); + this.name = 'Bad Request'; + this.status = 400; + this.code = 'BAD_REQUEST'; + } +} + +export class ServerError extends AxiosError { + constructor(message: string) { + super(message); + this.name = 'Server Error'; + this.status = 500; + this.code = 'SERVER_ERROR'; + } +} diff --git a/src/jobs/ega/types/common.ts b/src/jobs/ega/types/common.ts index 1900491..4c71527 100644 --- a/src/jobs/ega/types/common.ts +++ b/src/jobs/ega/types/common.ts @@ -29,7 +29,7 @@ export type DateString = z.infer; // For ISO8601 Datetime strings (i.e. '2021-01-01T00:00:00.000Z') // Note: for safeParse to allow the +00:00, as in '2024-01-31T16:25:13.725724+00:00', would need .datetime({ offset: true }) -export const DateTime = z.string().datetime(); +export const DateTime = z.string().datetime({ offset: true }); export type DateTime = z.infer; /* ******************* * diff --git a/src/jobs/ega/types/responses.ts b/src/jobs/ega/types/responses.ts index 13dc2eb..35bf81c 100644 --- a/src/jobs/ega/types/responses.ts +++ b/src/jobs/ega/types/responses.ts @@ -52,7 +52,7 @@ export type Dac = z.infer; export const Dataset = z.object({ accession_id: DatasetAccessionId, - title: z.string(), + title: z.string().nullable(), // TODO: verify this is expected description: z.string().optional(), }); export type Dataset = z.infer; @@ -75,9 +75,9 @@ export const EgaPermissionRequest = z.object({ // TODO: api docs state this should be a DateTime string, but receiving 'YYYY-MM-DD` string. May need to change to coerceable date? date: DateString, username: z.string(), - full_name: z.string(), - email: z.string().email(), - organisation: z.string(), + full_name: z.string().nullable(), + email: z.string().email().nullable(), + organisation: z.string().nullable(), dataset_accession_id: DatasetAccessionId, dataset_title: z.string().nullable(), dac_accession_id: DacAccessionId, @@ -101,7 +101,7 @@ export type ApprovePermissionResponse = z.infer; -export const EgaDacoUser = EgaUser.merge(z.object({ appExpiry: DateTime, appId: z.string() })); +export const EgaDacoUser = EgaUser.merge(z.object({ appExpiry: z.date(), appId: z.string() })); export type EgaDacoUser = z.infer; export type EgaDacoUserMap = Record; diff --git a/src/jobs/ega/types/results.ts b/src/jobs/ega/types/results.ts index 9ba2b55..563b830 100644 --- a/src/jobs/ega/types/results.ts +++ b/src/jobs/ega/types/results.ts @@ -86,5 +86,8 @@ export type CreatePermissionRequestsFailure = ServerError; export type ApprovedPermissionRequestsFailure = | ServerError | 'INVALID_APPROVE_PERMISSION_REQUESTS_RESPONSE'; -export type RevokePermissionsFailure = ServerError | 'INVALID_REVOKE_PERMISSIONS_RESPONSE'; +export type RevokePermissionsFailure = + | ServerError + | 'INVALID_REVOKE_PERMISSIONS_RESPONSE' + | 'PERMISSION_DOES_NOT_EXIST'; export type GetUserFailure = ServerError | 'NOT_FOUND' | 'INVALID_USER'; diff --git a/src/jobs/ega/utils.ts b/src/jobs/ega/utils.ts index 9e0335d..eb5f090 100644 --- a/src/jobs/ega/utils.ts +++ b/src/jobs/ega/utils.ts @@ -20,7 +20,7 @@ import { uniqBy } from 'lodash'; import { UserDataFromApprovedApplicationsResult } from '../../domain/interface'; import { getUsersFromApprovedApps } from '../../domain/service/applications/search'; -import { DatasetAccessionId, DateTime } from './types/common'; +import { DatasetAccessionId } from './types/common'; import { ApprovePermissionRequest, PermissionRequest, RevokePermission } from './types/requests'; import { ApprovePermissionResponse, RevokePermissionResponse } from './types/responses'; @@ -93,11 +93,13 @@ export const createPermissionRequest = ( */ export const createPermissionApprovalRequest = ( permissionRequestId: number, - appExpiry: DateTime, -): ApprovePermissionRequest => ({ - request_id: permissionRequestId, - expires_at: appExpiry, -}); + appExpiry: Date, +): ApprovePermissionRequest => { + return { + request_id: permissionRequestId, + expires_at: appExpiry.toISOString(), + }; +}; /** * Create revoke permission request object for DELETE /requests diff --git a/src/jobs/egaPermissionsReconciliation.ts b/src/jobs/egaPermissionsReconciliation.ts index 575fefd..e57cba0 100644 --- a/src/jobs/egaPermissionsReconciliation.ts +++ b/src/jobs/egaPermissionsReconciliation.ts @@ -17,12 +17,13 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import { chunk } from 'lodash'; import { getAppConfig } from '../config'; import logger, { buildMessage } from '../logger'; import { egaApiClient, EgaClient } from './ega/egaClient'; import { DatasetAccessionId } from './ega/types/common'; -import { RevokePermission } from './ega/types/requests'; -import { Dataset, EgaDacoUserMap } from './ega/types/responses'; +import { PermissionRequest, RevokePermission } from './ega/types/requests'; +import { Dataset, EgaDacoUser, EgaDacoUserMap } from './ega/types/responses'; import { isSuccess } from './ega/types/results'; import { ApprovedUser, @@ -30,11 +31,15 @@ import { createPermissionRequest, createRevokePermissionRequest, getApprovedUsers, - verifyPermissionApprovals, } from './ega/utils'; const JOB_NAME = 'RECONCILE_EGA_PERMISSIONS'; +// API request constants +const DEFAULT_OFFSET = 50; +const DEFAULT_LIMIT = 50; +const EGA_MAX_REQUEST_SIZE = 2000; + /** * Retrieve EGA user data for each user on DACO approved list * @param client EgaClient @@ -47,7 +52,7 @@ const JOB_NAME = 'RECONCILE_EGA_PERMISSIONS'; * username: boysue@example.com, * email: boysue@example.com, * accession_id: EGAW00000009999, - * appExpiry: '2024-10-01T14:06:41.485Z', + * appExpiry: 2024-10-01T14:06:41.485Z, * appId: 'DACO-1' * } * ... @@ -62,14 +67,27 @@ const getUsers = async ( for await (const user of approvedUsers) { try { const egaUser = await client.getUser(user); - if (egaUser.status === 'SUCCESS') { - const { data } = egaUser; - const egaDacoUser = { - ...data, - appExpiry: user.appExpiry.toDateString(), - appId: user.appId, - }; - egaUsers[data.username] = egaDacoUser; + switch (egaUser.status) { + case 'SUCCESS': + const { data } = egaUser; + const egaDacoUser = { + ...data, + appExpiry: user.appExpiry, + appId: user.appId, + }; + egaUsers[data.username] = egaDacoUser; + break; + case 'NOT_FOUND': + logger.debug(`No user found for [${user.email}].`); + break; + case 'INVALID_USER': + logger.error(`Invalid user: ${egaUser.message}`); + break; + case 'SERVER_ERROR': + logger.error(`Server error: ${egaUser.message}`); + break; + default: + logger.error('Unexpected error fetching user'); } } catch (err) { logger.error(err); @@ -78,15 +96,71 @@ const getUsers = async ( return egaUsers; }; +const handlePermissionRequests = async ( + egaClient: EgaClient, + approvedUser: EgaDacoUser, + permissionRequests: PermissionRequest[], +) => { + logger.debug(`There are ${permissionRequests.length} permissions needed.`); + // POST all requests + const chunkedPermissionRequests = chunk(permissionRequests, EGA_MAX_REQUEST_SIZE); + for await (const requests of chunkedPermissionRequests) { + const createRequestsResponse = await egaClient.createPermissionRequests(requests); + + if (isSuccess(createRequestsResponse)) { + // create approval requests objs + send all + if (createRequestsResponse.data.success.length) { + const approvalRequests = createRequestsResponse.data.success.map((request) => + createPermissionApprovalRequest(request.request_id, approvedUser.appExpiry), + ); + const approvePermissionRequestsResponse = await egaClient.approvePermissionRequests( + approvalRequests, + ); + if (isSuccess(approvePermissionRequestsResponse)) { + logger.debug( + `${approvePermissionRequestsResponse.data.num_granted} of ${requests.length} approval requests completed.`, + ); + } else { + logger.error( + `ApprovalRequests failed due to: ${approvePermissionRequestsResponse.message}`, + ); + } + } else { + console.log( + `Failures from create permission requests`, + createRequestsResponse.data.failure, + ); + } + } else { + logger.error( + `Request to create PermissionRequests failed due to: ${createRequestsResponse.message}`, + ); + } + } +}; +/** + * Process missing permissions for users on DACO ApprovedList, for each Dataset in the ICGC DAC + * Iterates through each user: + * 1) For each dataset: + * a) queries /permissions endpoint by datasetAccessionId + userId + * b) If no permission is found, creates PermissionRequest object and adds to permissionsRequest list + * 2) If there are items in the permissionsRequest list, divides requests into EGA_MAX_REQUEST_SIZE chunks + * For each chunk: + * a) Sends requests to POST /requests to create a PermissionRequest for each item + * b) Creates an ApprovePermissionRequest for each PermissionRequest received in the response from (a) + * c) Sends approval requests to PUT /requests + * @param egaClient + * @param egaUsers + * @param datasets + */ const processPermissionsForApprovedUsers = async ( egaClient: EgaClient, egaUsers: EgaDacoUserMap, datasets: Dataset[], ) => { const userList = Object.values(egaUsers); - for await (const approvedUser of userList) { - const permissionRequests = []; + const permissionRequests: PermissionRequest[] = []; for await (const dataset of datasets) { // check for existing permission const existingPermission = await egaClient.getPermissionByDatasetAndUserId( @@ -102,32 +176,17 @@ const processPermissionsForApprovedUsers = async ( ); permissionRequests.push(permissionRequest); } + } else { + logger.info(`Error fetching existing permission: ${existingPermission.message}`); } } if (permissionRequests.length) { - // POST all requests - const createRequestsResponse = await egaClient.createPermissionRequests(permissionRequests); - if (!isSuccess(createRequestsResponse)) { - throw new Error('Failed to create permissions requests'); - } - // create approval requests objs + send all - const approvalRequests = createRequestsResponse.data.success.map((request) => - createPermissionApprovalRequest(request.request_id, approvedUser.appExpiry), - ); - const approvePermissionRequestsResponse = await egaClient.approvePermissionRequests( - approvalRequests, - ); - if (isSuccess(approvePermissionRequestsResponse)) { - verifyPermissionApprovals(approvalRequests.length, approvePermissionRequestsResponse.data); - } + await handlePermissionRequests(egaClient, approvedUser, permissionRequests); } } logger.info('Completed processing permissions for all DACO approved users.'); }; -const DEFAULT_OFFSET = 50; -const DEFAULT_LIMIT = 50; - /** * Paginates through all permissions for a dataset and revokes permissions for users not found in approvedUsers * @param client EgaClient @@ -140,6 +199,7 @@ export const processPermissionsForDataset = async ( datasetAccessionId: DatasetAccessionId, approvedUsers: EgaDacoUserMap, ): Promise => { + let permissionsSet: Set = new Set(); let permissionsToRevoke: RevokePermission[] = []; let offset = 0; let limit = DEFAULT_LIMIT; @@ -158,25 +218,42 @@ export const processPermissionsForDataset = async ( // check if permission username is found in approvedUsers const hasAccess = approvedUsers[permission.username]; if (!hasAccess) { - const revokeRequest = createRevokePermissionRequest(permission.permission_id); - permissionsToRevoke.push(revokeRequest); + permissionsSet.add(permission.permission_id); } }); - offset = offset + DEFAULT_OFFSET; const totalResults = permissionsFailures.length + permissionsSuccesses.length; - paging = totalResults >= limit; - } - } - if (permissionsToRevoke.length) { - const revokeResponse = await client.revokePermissions(permissionsToRevoke); - if (isSuccess(revokeResponse)) { - logger.info( - `Successfully revoked ${revokeResponse.data.num_revoked} of total ${permissionsToRevoke.length} permissions for DATASET ${datasetAccessionId}.`, - ); + paging = totalResults === limit; + // TODO: there is a repeated permission result when paginating, + // subtracting 1 from the offset prevents paging from stopping before all unique results are retrieved + offset = offset + DEFAULT_OFFSET - 1; } else { logger.error( - `There was an error revoking permissions for DATASET ${datasetAccessionId} - ${revokeResponse.message}.`, + `GET permissions for dataset ${datasetAccessionId} failed - ${permissions.message}`, ); + // stop paging results if request completely fails to prevent endless loop + // can a retry mechanism be added here, if error is retryable? + paging = false; + } + } + const setSize = permissionsSet.size; + if (setSize > 0) { + logger.debug(`There are ${permissionsSet.size} permissions to remove.`); + permissionsSet.forEach((perm) => { + const revokeReq = createRevokePermissionRequest(perm); + permissionsToRevoke.push(revokeReq); + }); + const chunkedRevokeRequests = chunk(permissionsToRevoke, EGA_MAX_REQUEST_SIZE); + for await (const requests of chunkedRevokeRequests) { + const revokeResponse = await client.revokePermissions(requests); + if (isSuccess(revokeResponse)) { + logger.info( + `Successfully revoked ${revokeResponse.data.num_revoked} of total ${setSize} permissions for DATASET ${datasetAccessionId}.`, + ); + } else { + logger.error( + `There was an error revoking permissions for DATASET ${datasetAccessionId} - ${revokeResponse.message}.`, + ); + } } } else { logger.info(`There are no permissions to revoke for DATASET ${datasetAccessionId}.`); @@ -191,7 +268,7 @@ export const processPermissionsForDataset = async ( * 4) Create permissions, on each dataset, for each user on the DACO approved list, if no existing permission is found * 5) Process existing permissions for each dataset + revoke those which belong to users not on the DACO approved list */ -export default async function () { +async function runEgaPermissionsReconciliation() { // retrieve approved users list from daco system const dacoUsers = await getApprovedUsers(); // initialize EGA Axios client @@ -208,9 +285,12 @@ export default async function () { // TODO: retry here? throw new Error('Failed to fetch datasets'); } + logger.debug(`Successfully retrieved ${datasets.data.success.length} for DAC ${dacId}.`); // retrieve corresponding users in EGA system const egaUsers = await getUsers(egaClient, dacoUsers); + logger.debug(`Retrieved ${Object.keys(egaUsers).length} corresponding users from EGA.`); const datasetsRetrieved = datasets.data.success; + // check DACO approved users have expected EGA permissions for each dataset await processPermissionsForApprovedUsers(egaClient, egaUsers, datasetsRetrieved); @@ -222,4 +302,7 @@ export default async function () { } logger.info(buildMessage(JOB_NAME, 'Completed.')); + return 'OK'; } + +export default runEgaPermissionsReconciliation; From 04879b4049752e4a659632d3e99b90e60b265c9b Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Thu, 3 Oct 2024 14:01:05 -0400 Subject: [PATCH 20/25] Expand tsdocs --- src/jobs/egaPermissionsReconciliation.ts | 46 +++++++++++++----------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/jobs/egaPermissionsReconciliation.ts b/src/jobs/egaPermissionsReconciliation.ts index e57cba0..a069847 100644 --- a/src/jobs/egaPermissionsReconciliation.ts +++ b/src/jobs/egaPermissionsReconciliation.ts @@ -96,19 +96,23 @@ const getUsers = async ( return egaUsers; }; -const handlePermissionRequests = async ( +/** + * Function to create + approve a list of PermissionsRequests + * 1) Sends requests to POST /requests to create a PermissionRequest for each item + * 2) Creates an ApprovePermissionRequest for each PermissionRequest received in the list response from (1) + * 3) Sends all ApprovePermissionRequests to PUT /requests + * @param egaClient EgaClient + * @param approvedUser EgaDacoUser + * @param permissionRequests PermissionRequest[] + */ +const createRequiredPermissions = async ( egaClient: EgaClient, approvedUser: EgaDacoUser, - permissionRequests: PermissionRequest[], + requests: PermissionRequest[], ) => { - logger.debug(`There are ${permissionRequests.length} permissions needed.`); - // POST all requests - const chunkedPermissionRequests = chunk(permissionRequests, EGA_MAX_REQUEST_SIZE); - for await (const requests of chunkedPermissionRequests) { - const createRequestsResponse = await egaClient.createPermissionRequests(requests); - - if (isSuccess(createRequestsResponse)) { - // create approval requests objs + send all + const createRequestsResponse = await egaClient.createPermissionRequests(requests); + switch (createRequestsResponse.status) { + case 'SUCCESS': if (createRequestsResponse.data.success.length) { const approvalRequests = createRequestsResponse.data.success.map((request) => createPermissionApprovalRequest(request.request_id, approvedUser.appExpiry), @@ -131,24 +135,23 @@ const handlePermissionRequests = async ( createRequestsResponse.data.failure, ); } - } else { + break; + case 'SERVER_ERROR': + default: logger.error( `Request to create PermissionRequests failed due to: ${createRequestsResponse.message}`, ); - } } }; + /** - * Process missing permissions for users on DACO ApprovedList, for each Dataset in the ICGC DAC + * Process any missing permissions for all users on DACO ApprovedList, for each Dataset in the ICGC DAC * Iterates through each user: * 1) For each dataset: - * a) queries /permissions endpoint by datasetAccessionId + userId + * a) queries GET dacs/{dacId}/permissions endpoint by datasetAccessionId + userId * b) If no permission is found, creates PermissionRequest object and adds to permissionsRequest list * 2) If there are items in the permissionsRequest list, divides requests into EGA_MAX_REQUEST_SIZE chunks - * For each chunk: - * a) Sends requests to POST /requests to create a PermissionRequest for each item - * b) Creates an ApprovePermissionRequest for each PermissionRequest received in the response from (a) - * c) Sends approval requests to PUT /requests + * a) For each chunk, creates permissions with createRequiredPermissions() call * @param egaClient * @param egaUsers * @param datasets @@ -181,7 +184,10 @@ const processPermissionsForApprovedUsers = async ( } } if (permissionRequests.length) { - await handlePermissionRequests(egaClient, approvedUser, permissionRequests); + const chunkedPermissionRequests = chunk(permissionRequests, EGA_MAX_REQUEST_SIZE); + for await (const requests of chunkedPermissionRequests) { + await createRequiredPermissions(egaClient, approvedUser, requests); + } } } logger.info('Completed processing permissions for all DACO approved users.'); @@ -290,7 +296,7 @@ async function runEgaPermissionsReconciliation() { const egaUsers = await getUsers(egaClient, dacoUsers); logger.debug(`Retrieved ${Object.keys(egaUsers).length} corresponding users from EGA.`); const datasetsRetrieved = datasets.data.success; - + logger.debug(`Retrieved ${datasetsRetrieved.length} datasets for ${dacId}.`); // check DACO approved users have expected EGA permissions for each dataset await processPermissionsForApprovedUsers(egaClient, egaUsers, datasetsRetrieved); From 951bebc00928354b8d24ff1eca474dadc69dc0ca Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Thu, 3 Oct 2024 14:23:56 -0400 Subject: [PATCH 21/25] add comment for api bug, reorg ega services --- src/jobs/ega/egaPermissionsReconciliation.ts | 78 ++++++++++ .../services/permissions.ts} | 138 ++---------------- src/jobs/ega/services/users.ts | 79 ++++++++++ src/jobs/ega/types/constants.ts | 23 +++ 4 files changed, 194 insertions(+), 124 deletions(-) create mode 100644 src/jobs/ega/egaPermissionsReconciliation.ts rename src/jobs/{egaPermissionsReconciliation.ts => ega/services/permissions.ts} (64%) create mode 100644 src/jobs/ega/services/users.ts create mode 100644 src/jobs/ega/types/constants.ts diff --git a/src/jobs/ega/egaPermissionsReconciliation.ts b/src/jobs/ega/egaPermissionsReconciliation.ts new file mode 100644 index 0000000..42d50ac --- /dev/null +++ b/src/jobs/ega/egaPermissionsReconciliation.ts @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { getAppConfig } from '../../config'; +import logger, { buildMessage } from '../../logger'; +import { egaApiClient } from './egaClient'; +import { + processPermissionsForApprovedUsers, + processPermissionsForDataset, +} from './services/permissions'; +import { getUsers } from './services/users'; +import { isSuccess } from './types/results'; +import { getApprovedUsers } from './utils'; + +const JOB_NAME = 'RECONCILE_EGA_PERMISSIONS'; + +/** + * Steps: + * 1) Retrieve approved users list from dac db + * 2) Retrieve datasets for DAC + * 3) Retrieve corresponding list of users from EGA API + * 4) Create permissions, on each dataset, for each user on the DACO approved list, if no existing permission is found + * 5) Process existing permissions for each dataset + revoke those which belong to users not on the DACO approved list + */ +async function runEgaPermissionsReconciliation() { + // retrieve approved users list from daco system + const dacoUsers = await getApprovedUsers(); + // initialize EGA Axios client + const egaClient = await egaApiClient(); + + // retrieve all datasets for ICGC DAC + const { + ega: { dacId }, + } = getAppConfig(); + const datasets = await egaClient.getDatasetsForDac(dacId); + + // get datasets failed completely + if (!isSuccess(datasets)) { + // TODO: retry here? + throw new Error('Failed to fetch datasets'); + } + logger.debug(`Successfully retrieved ${datasets.data.success.length} for DAC ${dacId}.`); + // retrieve corresponding users in EGA system + const egaUsers = await getUsers(egaClient, dacoUsers); + logger.debug(`Retrieved ${Object.keys(egaUsers).length} corresponding users from EGA.`); + const datasetsRetrieved = datasets.data.success; + logger.debug(`Retrieved ${datasetsRetrieved.length} datasets for ${dacId}.`); + // check DACO approved users have expected EGA permissions for each dataset + await processPermissionsForApprovedUsers(egaClient, egaUsers, datasetsRetrieved); + + // can add a return value to these process functions if needed, i.e. BatchJobReport + + // Check existing permissions per dataset + revoke if needed + for await (const dataset of datasetsRetrieved) { + await processPermissionsForDataset(egaClient, dataset.accession_id, egaUsers); + } + + logger.info(buildMessage(JOB_NAME, 'Completed.')); + return 'OK'; +} + +export default runEgaPermissionsReconciliation; diff --git a/src/jobs/egaPermissionsReconciliation.ts b/src/jobs/ega/services/permissions.ts similarity index 64% rename from src/jobs/egaPermissionsReconciliation.ts rename to src/jobs/ega/services/permissions.ts index a069847..1fcf18f 100644 --- a/src/jobs/egaPermissionsReconciliation.ts +++ b/src/jobs/ega/services/permissions.ts @@ -18,83 +18,18 @@ */ import { chunk } from 'lodash'; -import { getAppConfig } from '../config'; -import logger, { buildMessage } from '../logger'; -import { egaApiClient, EgaClient } from './ega/egaClient'; -import { DatasetAccessionId } from './ega/types/common'; -import { PermissionRequest, RevokePermission } from './ega/types/requests'; -import { Dataset, EgaDacoUser, EgaDacoUserMap } from './ega/types/responses'; -import { isSuccess } from './ega/types/results'; +import logger from '../../../logger'; +import { EgaClient } from '../egaClient'; +import { DatasetAccessionId } from '../types/common'; +import { DEFAULT_LIMIT, DEFAULT_OFFSET, EGA_MAX_REQUEST_SIZE } from '../types/constants'; +import { PermissionRequest, RevokePermission } from '../types/requests'; +import { Dataset, EgaDacoUser, EgaDacoUserMap } from '../types/responses'; +import { isSuccess } from '../types/results'; import { - ApprovedUser, createPermissionApprovalRequest, createPermissionRequest, createRevokePermissionRequest, - getApprovedUsers, -} from './ega/utils'; - -const JOB_NAME = 'RECONCILE_EGA_PERMISSIONS'; - -// API request constants -const DEFAULT_OFFSET = 50; -const DEFAULT_LIMIT = 50; -const EGA_MAX_REQUEST_SIZE = 2000; - -/** - * Retrieve EGA user data for each user on DACO approved list - * @param client EgaClient - * @param dacoUsers ApprovedUser[] - * @returns EgaDacoUserMap - * @example - * // returns { - * boysue@example.com: { - * id: 123, - * username: boysue@example.com, - * email: boysue@example.com, - * accession_id: EGAW00000009999, - * appExpiry: 2024-10-01T14:06:41.485Z, - * appId: 'DACO-1' - * } - * ... - * } - * getUsers(client, approvedUsersList) - */ -const getUsers = async ( - client: EgaClient, - approvedUsers: ApprovedUser[], -): Promise => { - let egaUsers: EgaDacoUserMap = {}; - for await (const user of approvedUsers) { - try { - const egaUser = await client.getUser(user); - switch (egaUser.status) { - case 'SUCCESS': - const { data } = egaUser; - const egaDacoUser = { - ...data, - appExpiry: user.appExpiry, - appId: user.appId, - }; - egaUsers[data.username] = egaDacoUser; - break; - case 'NOT_FOUND': - logger.debug(`No user found for [${user.email}].`); - break; - case 'INVALID_USER': - logger.error(`Invalid user: ${egaUser.message}`); - break; - case 'SERVER_ERROR': - logger.error(`Server error: ${egaUser.message}`); - break; - default: - logger.error('Unexpected error fetching user'); - } - } catch (err) { - logger.error(err); - } - } - return egaUsers; -}; +} from '../utils'; /** * Function to create + approve a list of PermissionsRequests @@ -105,7 +40,7 @@ const getUsers = async ( * @param approvedUser EgaDacoUser * @param permissionRequests PermissionRequest[] */ -const createRequiredPermissions = async ( +export const createRequiredPermissions = async ( egaClient: EgaClient, approvedUser: EgaDacoUser, requests: PermissionRequest[], @@ -156,7 +91,7 @@ const createRequiredPermissions = async ( * @param egaUsers * @param datasets */ -const processPermissionsForApprovedUsers = async ( +export const processPermissionsForApprovedUsers = async ( egaClient: EgaClient, egaUsers: EgaDacoUserMap, datasets: Dataset[], @@ -229,8 +164,10 @@ export const processPermissionsForDataset = async ( }); const totalResults = permissionsFailures.length + permissionsSuccesses.length; paging = totalResults === limit; - // TODO: there is a repeated permission result when paginating, - // subtracting 1 from the offset prevents paging from stopping before all unique results are retrieved + // TODO: there is a bug in the GET /permissions and GET /dacs/{dacId}/permissions result when paginating + // Any request that includes the 19th element of the result array will have the final element in the array is replaced by the first element from the sorted dataset + // In practice this means that paging will stop before all unique elements are returned, as some of the total is made up of these duplicate values + // Temp solution is to subtract 1 from the offset (limit - 1), which "backtracks" the paging to ensure the element that gets missed in the last array position is captured offset = offset + DEFAULT_OFFSET - 1; } else { logger.error( @@ -265,50 +202,3 @@ export const processPermissionsForDataset = async ( logger.info(`There are no permissions to revoke for DATASET ${datasetAccessionId}.`); } }; - -/** - * Steps: - * 1) Retrieve approved users list from dac db - * 2) Retrieve datasets for DAC - * 3) Retrieve corresponding list of users from EGA API - * 4) Create permissions, on each dataset, for each user on the DACO approved list, if no existing permission is found - * 5) Process existing permissions for each dataset + revoke those which belong to users not on the DACO approved list - */ -async function runEgaPermissionsReconciliation() { - // retrieve approved users list from daco system - const dacoUsers = await getApprovedUsers(); - // initialize EGA Axios client - const egaClient = await egaApiClient(); - - // retrieve all datasets for ICGC DAC - const { - ega: { dacId }, - } = getAppConfig(); - const datasets = await egaClient.getDatasetsForDac(dacId); - - // get datasets failed completely - if (!isSuccess(datasets)) { - // TODO: retry here? - throw new Error('Failed to fetch datasets'); - } - logger.debug(`Successfully retrieved ${datasets.data.success.length} for DAC ${dacId}.`); - // retrieve corresponding users in EGA system - const egaUsers = await getUsers(egaClient, dacoUsers); - logger.debug(`Retrieved ${Object.keys(egaUsers).length} corresponding users from EGA.`); - const datasetsRetrieved = datasets.data.success; - logger.debug(`Retrieved ${datasetsRetrieved.length} datasets for ${dacId}.`); - // check DACO approved users have expected EGA permissions for each dataset - await processPermissionsForApprovedUsers(egaClient, egaUsers, datasetsRetrieved); - - // can add a return value to these process functions if needed, i.e. BatchJobReport - - // Check existing permissions per dataset + revoke if needed - for await (const dataset of datasetsRetrieved) { - await processPermissionsForDataset(egaClient, dataset.accession_id, egaUsers); - } - - logger.info(buildMessage(JOB_NAME, 'Completed.')); - return 'OK'; -} - -export default runEgaPermissionsReconciliation; diff --git a/src/jobs/ega/services/users.ts b/src/jobs/ega/services/users.ts new file mode 100644 index 0000000..2e48dbe --- /dev/null +++ b/src/jobs/ega/services/users.ts @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import logger from '../../../logger'; +import { EgaClient } from '../egaClient'; +import { EgaDacoUserMap } from '../types/responses'; +import { ApprovedUser } from '../utils'; + +/** + * Retrieve EGA user data for each user on DACO approved list + * @param client EgaClient + * @param dacoUsers ApprovedUser[] + * @returns EgaDacoUserMap + * @example + * // returns { + * boysue@example.com: { + * id: 123, + * username: boysue@example.com, + * email: boysue@example.com, + * accession_id: EGAW00000009999, + * appExpiry: 2024-10-01T14:06:41.485Z, + * appId: 'DACO-1' + * } + * ... + * } + * getUsers(client, approvedUsersList) + */ +export const getUsers = async ( + client: EgaClient, + approvedUsers: ApprovedUser[], +): Promise => { + let egaUsers: EgaDacoUserMap = {}; + for await (const user of approvedUsers) { + try { + const egaUser = await client.getUser(user); + switch (egaUser.status) { + case 'SUCCESS': + const { data } = egaUser; + const egaDacoUser = { + ...data, + appExpiry: user.appExpiry, + appId: user.appId, + }; + egaUsers[data.username] = egaDacoUser; + break; + case 'NOT_FOUND': + logger.debug(`No user found for [${user.email}].`); + break; + case 'INVALID_USER': + logger.error(`Invalid user: ${egaUser.message}`); + break; + case 'SERVER_ERROR': + logger.error(`Server error: ${egaUser.message}`); + break; + default: + logger.error('Unexpected error fetching user'); + } + } catch (err) { + logger.error(err); + } + } + return egaUsers; +}; diff --git a/src/jobs/ega/types/constants.ts b/src/jobs/ega/types/constants.ts new file mode 100644 index 0000000..1cff10b --- /dev/null +++ b/src/jobs/ega/types/constants.ts @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// API request constants +export const DEFAULT_LIMIT = 50; +export const DEFAULT_OFFSET = DEFAULT_LIMIT; +export const EGA_MAX_REQUEST_SIZE = 2000; From f4be3e895636122b93f55d80631e830116263003 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Thu, 3 Oct 2024 14:41:07 -0400 Subject: [PATCH 22/25] move custom errors to types dir --- src/jobs/ega/egaClient.ts | 2 +- src/jobs/ega/{ => types}/errors.ts | 0 src/jobs/ega/utils.ts | 10 ++++++++-- 3 files changed, 9 insertions(+), 3 deletions(-) rename src/jobs/ega/{ => types}/errors.ts (100%) diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts index 4f998cf..1b30994 100644 --- a/src/jobs/ega/egaClient.ts +++ b/src/jobs/ega/egaClient.ts @@ -29,7 +29,7 @@ import { EGA_REALMS_PATH, EGA_TOKEN_ENDPOINT, } from '../../utils/constants'; -import { BadRequestError, NotFoundError, ServerError, TooManyRequestsError } from './errors'; +import { BadRequestError, NotFoundError, ServerError, TooManyRequestsError } from './types/errors'; import { DacAccessionId, DatasetAccessionId } from './types/common'; import { ApprovePermissionRequest, PermissionRequest, RevokePermission } from './types/requests'; import { diff --git a/src/jobs/ega/errors.ts b/src/jobs/ega/types/errors.ts similarity index 100% rename from src/jobs/ega/errors.ts rename to src/jobs/ega/types/errors.ts diff --git a/src/jobs/ega/utils.ts b/src/jobs/ega/utils.ts index eb5f090..3093499 100644 --- a/src/jobs/ega/utils.ts +++ b/src/jobs/ega/utils.ts @@ -124,7 +124,10 @@ export const getErrorMessage = (error: unknown, defaultMessage: string): string error instanceof Error ? error.message : defaultMessage; /** - * + * Verify total permission approvals sent in request matches response num_granted + * @param numRequests number - length of permissionsRequests array + * @param approvalResponse ApprovePermissionResponse + * @returns boolean */ export const verifyPermissionApprovals = ( numRequests: number, @@ -132,7 +135,10 @@ export const verifyPermissionApprovals = ( ): boolean => numRequests === approvalResponse.num_granted; /** - * + * Verify total permission re sent in request matches response num_revoked + * @param numRequests number - length of permissionsRequests array + * @param approvalResponse RevokePermissionResponse + * @returns boolean */ export const verifyPermissionRevocations = ( numRequests: number, From 816df1920910eadff72133b8917517eaf8b4db0f Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Fri, 4 Oct 2024 12:49:15 -0400 Subject: [PATCH 23/25] =?UTF-8?q?=E2=9E=95=20Add=20pThrottle=20dependency?= =?UTF-8?q?=20as=20source=20code,=20rate=20limit=20ega=20client=20funcs=20?= =?UTF-8?q?(#461)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add pThrottle source code dir, add throttling to all egaClient functions --- .env.example | 2 + README.md | 20 ++++--- pThrottle/index.d.ts | 119 ++++++++++++++++++++++++++++++++++++++ pThrottle/index.js | 114 ++++++++++++++++++++++++++++++++++++ pThrottle/license | 9 +++ pThrottle/readme.md | 20 +++++++ src/config.ts | 4 ++ src/jobs/ega/egaClient.ts | 23 +++++--- tsconfig.json | 2 +- 9 files changed, 295 insertions(+), 18 deletions(-) create mode 100644 pThrottle/index.d.ts create mode 100644 pThrottle/index.js create mode 100644 pThrottle/license create mode 100644 pThrottle/readme.md diff --git a/.env.example b/.env.example index 57592ce..bf89676 100644 --- a/.env.example +++ b/.env.example @@ -120,3 +120,5 @@ EGA_API_URL= EGA_USERNAME= EGA_PASSWORD= DAC_ID= +EGA_MAX_REQUEST_LIMIT=3; +EGA_MAX_REQUEST_INTERVAL=1000; # in milliseconds diff --git a/README.md b/README.md index 886c486..0bf43a9 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,17 @@ Development of the Data Access Control API ## Environment Variables -| Name | Description | Type | Required | Default | -| ------------------- | ----------------------------------------------------------------------------- | -------- | -------- | ------- | -| EGA_CLIENT_ID | Client ID for EGA API | `string` | true | | -| EGA_AUTH_HOST | Root URL for EGA authentication server | `string` | true | | -| EGA_AUTH_REALM_NAME | Realm name for EGA authentication server | `string` | true | | -| EGA_API_URL | Root URL for EGA API | `string` | true | | -| EGA_USERNAME | Username for account used to gain access token from EGA authentication server | `string` | true | | -| EGA_PASSWORD | Password for account used to gain access token from EGA authentication server | `string` | true | | -| DAC_ID | AccessionId for ICGC DAC | `string` | true | | +| Name | Description | Type | Required | Default | +| ------------------------ | ---------------------------------------------------------------------------------------------------------- | -------- | -------- | ------- | +| EGA_CLIENT_ID | Client ID for EGA API | `string` | true | | +| EGA_AUTH_HOST | Root URL for EGA authentication server | `string` | true | | +| EGA_AUTH_REALM_NAME | Realm name for EGA authentication server | `string` | true | | +| EGA_API_URL | Root URL for EGA API | `string` | true | | +| EGA_USERNAME | Username for account used to gain access token from EGA authentication server | `string` | true | | +| EGA_PASSWORD | Password for account used to gain access token from EGA authentication server | `string` | true | | +| DAC_ID | AccessionId for ICGC DAC | `string` | true | | +| EGA_MAX_REQUEST_LIMIT | For EGA API rate limiting. The max number of API requests per interval value `EGA_MAX_REQUEST_INTERVAL` | `number` | true | 3 | +| EGA_MAX_REQUEST_INTERVAL | For EGA API rate limiting. Interval of time for API request limit `EGA_MAX_REQUEST_LIMIT`, in milliseconds | `number` | true | 1000 | ## Feature Flags diff --git a/pThrottle/index.d.ts b/pThrottle/index.d.ts new file mode 100644 index 0000000..4f79e55 --- /dev/null +++ b/pThrottle/index.d.ts @@ -0,0 +1,119 @@ +export class AbortError extends Error { + readonly name: 'AbortError'; + + private constructor(); +} + +type AnyFunction = (...arguments_: readonly any[]) => unknown; + +export type ThrottledFunction = F & { + /** + Whether future function calls should be throttled or count towards throttling thresholds. + + @default true + */ + isEnabled: boolean; + + /** + The number of queued items waiting to be executed. + */ + readonly queueSize: number; + + /** + Abort pending executions. All unresolved promises are rejected with a `pThrottle.AbortError` error. + */ + abort(): void; +}; + +export type Options = { + /** + The maximum number of calls within an `interval`. + */ + readonly limit: number; + + /** + The timespan for `limit` in milliseconds. + */ + readonly interval: number; + + /** + Use a strict, more resource intensive, throttling algorithm. The default algorithm uses a windowed approach that will work correctly in most cases, limiting the total number of calls at the specified limit per interval window. The strict algorithm throttles each call individually, ensuring the limit is not exceeded for any interval. + + @default false + */ + readonly strict?: boolean; + + /** + Get notified when function calls are delayed due to exceeding the `limit` of allowed calls within the given `interval`. The delayed call arguments are passed to the `onDelay` callback. + + Can be useful for monitoring the throttling efficiency. + + @example + ``` + import pThrottle from 'p-throttle'; + + const throttle = pThrottle({ + limit: 2, + interval: 1000, + onDelay: (a, b) => { + console.log(`Reached interval limit, call is delayed for ${a} ${b}`); + }, + }); + + const throttled = throttle((a, b) => { + console.log(`Executing with ${a} ${b}...`); + }); + + await throttled(1, 2); + await throttled(3, 4); + await throttled(5, 6); + //=> Executing with 1 2... + //=> Executing with 3 4... + //=> Reached interval limit, call is delayed for 5 6 + //=> Executing with 5 6... + ``` + */ + readonly onDelay?: (...arguments_: readonly any[]) => void; +}; + +/** +Throttle promise-returning/async/normal functions. + +It rate-limits function calls without discarding them, making it ideal for external API interactions where avoiding call loss is crucial. + +@returns A throttle function. + +Both the `limit` and `interval` options must be specified. + +@example +``` +import pThrottle from 'p-throttle'; + +const now = Date.now(); + +const throttle = pThrottle({ + limit: 2, + interval: 1000 +}); + +const throttled = throttle(async index => { + const secDiff = ((Date.now() - now) / 1000).toFixed(); + return `${index}: ${secDiff}s`; +}); + +for (let index = 1; index <= 6; index++) { + (async () => { + console.log(await throttled(index)); + })(); +} +//=> 1: 0s +//=> 2: 0s +//=> 3: 1s +//=> 4: 1s +//=> 5: 2s +//=> 6: 2s +``` +*/ +export default function pThrottle( + options: Options, +): (function_: F) => ThrottledFunction; diff --git a/pThrottle/index.js b/pThrottle/index.js new file mode 100644 index 0000000..453cf48 --- /dev/null +++ b/pThrottle/index.js @@ -0,0 +1,114 @@ +export class AbortError extends Error { + constructor() { + super('Throttled function aborted'); + this.name = 'AbortError'; + } +} + +export default function pThrottle({ limit, interval, strict, onDelay }) { + if (!Number.isFinite(limit)) { + throw new TypeError('Expected `limit` to be a finite number'); + } + + if (!Number.isFinite(interval)) { + throw new TypeError('Expected `interval` to be a finite number'); + } + + const queue = new Map(); + + let currentTick = 0; + let activeCount = 0; + + function windowedDelay() { + const now = Date.now(); + + if (now - currentTick > interval) { + activeCount = 1; + currentTick = now; + return 0; + } + + if (activeCount < limit) { + activeCount++; + } else { + currentTick += interval; + activeCount = 1; + } + + return currentTick - now; + } + + const strictTicks = []; + + function strictDelay() { + const now = Date.now(); + + // Clear the queue if there's a significant delay since the last execution + if (strictTicks.length > 0 && now - strictTicks.at(-1) > interval) { + strictTicks.length = 0; + } + + // If the queue is not full, add the current time and execute immediately + if (strictTicks.length < limit) { + strictTicks.push(now); + return 0; + } + + // Calculate the next execution time based on the first item in the queue + const nextExecutionTime = strictTicks[0] + interval; + + // Shift the queue and add the new execution time + strictTicks.shift(); + strictTicks.push(nextExecutionTime); + + // Calculate the delay for the current execution + return Math.max(0, nextExecutionTime - now); + } + + const getDelay = strict ? strictDelay : windowedDelay; + + return (function_) => { + const throttled = function (...arguments_) { + if (!throttled.isEnabled) { + return (async () => function_.apply(this, arguments_))(); + } + + let timeoutId; + return new Promise((resolve, reject) => { + const execute = () => { + resolve(function_.apply(this, arguments_)); + queue.delete(timeoutId); + }; + + const delay = getDelay(); + if (delay > 0) { + timeoutId = setTimeout(execute, delay); + queue.set(timeoutId, reject); + onDelay?.(...arguments_); + } else { + execute(); + } + }); + }; + + throttled.abort = () => { + for (const timeout of queue.keys()) { + clearTimeout(timeout); + queue.get(timeout)(new AbortError()); + } + + queue.clear(); + strictTicks.splice(0, strictTicks.length); + }; + + throttled.isEnabled = true; + + Object.defineProperty(throttled, 'queueSize', { + get() { + return queue.size; + }, + }); + + return throttled; + }; +} diff --git a/pThrottle/license b/pThrottle/license new file mode 100644 index 0000000..fa7ceba --- /dev/null +++ b/pThrottle/license @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pThrottle/readme.md b/pThrottle/readme.md new file mode 100644 index 0000000..a5c59d4 --- /dev/null +++ b/pThrottle/readme.md @@ -0,0 +1,20 @@ +# p-throttle + +> Throttle promise-returning & async functions + +Copied from [p-throttle]('https://github.com/sindresorhus/p-throttle') for use with commonjs modules. Full README is available there or on [NPM](https://www.npmjs.com/package/p-throttle). + +To verify throttling is being applied, you can add a log to the `onDelay` option of the pThrottle configuration: + +``` +const throttle = pThrottle({ + limit: 2, // number of requests + interval: 1000, // time interval for limit + // a, b as args from the function being throttled + onDelay: (a, b) => { + console.log(`Reached interval limit, call is delayed for ${a} ${b}`); + }, +}) +``` + +> **Note:** This code was copied from Github (as of 24/10/03), so it is not necessarily up to date with the original source library. diff --git a/src/config.ts b/src/config.ts index 939741f..8962101 100644 --- a/src/config.ts +++ b/src/config.ts @@ -96,6 +96,8 @@ export interface AppConfig { authRealmName: string; apiUrl: string; dacId: string; + maxRequestLimit: number; + maxRequestInterval: number; }; } @@ -213,6 +215,8 @@ const buildAppContext = (): AppConfig => { authRealmName: checkIsDefined(process.env.EGA_AUTH_REALM_NAME), apiUrl: checkIsDefined(process.env.EGA_API_URL), dacId: checkIsDefined(process.env.DAC_ID), + maxRequestLimit: Number(process.env.EGA_MAX_REQUEST_LIMIT) || 3, + maxRequestInterval: Number(process.env.EGA_MAX_REQUEST_INTERVAL) || 1000, }, }; return config; diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts index 1b30994..4f657d1 100644 --- a/src/jobs/ega/egaClient.ts +++ b/src/jobs/ega/egaClient.ts @@ -55,6 +55,7 @@ import { } from './types/results'; import { safeParseArray, ZodResultAccumulator } from './types/zodSafeParseArray'; import { ApprovedUser, getErrorMessage } from './utils'; +import pThrottle from '../../../pThrottle'; const { DACS, DATASETS, PERMISSIONS, REQUESTS, USERS } = EGA_API; @@ -156,10 +157,16 @@ const refreshAccessToken = async (token: IdpToken): Promise => { */ export const egaApiClient = async () => { const { - ega: { dacId }, + ega: { dacId, maxRequestLimit, maxRequestInterval }, } = getAppConfig(); const token = await getAccessToken(); + // rate limit requests to a maximum of 3 per 1 second + const throttle = pThrottle({ + limit: maxRequestLimit, + interval: maxRequestInterval, + }); + apiAxiosClient.defaults.headers.common['Authorization'] = `Bearer ${token.access_token}`; apiAxiosClient.interceptors.response.use( @@ -450,13 +457,13 @@ export const egaApiClient = async () => { }; return { - approvePermissionRequests, - createPermissionRequests, - getDatasetsForDac, - getPermissionByDatasetAndUserId, - getPermissionsForDataset, - getUser, - revokePermissions, + approvePermissionRequests: throttle(approvePermissionRequests), + createPermissionRequests: throttle(createPermissionRequests), + getDatasetsForDac: throttle(getDatasetsForDac), + getPermissionByDatasetAndUserId: throttle(getPermissionByDatasetAndUserId), + getPermissionsForDataset: throttle(getPermissionsForDataset), + getUser: throttle(getUser), + revokePermissions: throttle(revokePermissions), }; }; diff --git a/tsconfig.json b/tsconfig.json index 9156c18..7ea06ac 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -21,5 +21,5 @@ } }, "exclude": ["node_modules"], - "include": ["src/**/*", "src/test/**/*"] + "include": ["src/**/*", "src/test/**/*", "pThrottle"] } From 947676789a2659fc0b063ca6fd57b097f0686683 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Fri, 11 Oct 2024 11:21:27 -0400 Subject: [PATCH 24/25] rename types + util functions for clarity --- src/jobs/ega/egaClient.ts | 8 ++--- src/jobs/ega/egaPermissionsReconciliation.ts | 8 ++--- src/jobs/ega/services/permissions.ts | 33 +++++++++++--------- src/jobs/ega/services/users.ts | 4 +-- src/jobs/ega/types/responses.ts | 4 +-- src/jobs/ega/utils.ts | 4 +-- 6 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/jobs/ega/egaClient.ts b/src/jobs/ega/egaClient.ts index 4f657d1..2cbd34f 100644 --- a/src/jobs/ega/egaClient.ts +++ b/src/jobs/ega/egaClient.ts @@ -34,7 +34,7 @@ import { DacAccessionId, DatasetAccessionId } from './types/common'; import { ApprovePermissionRequest, PermissionRequest, RevokePermission } from './types/requests'; import { ApprovePermissionResponse, - Dataset, + EgaDataset, EgaPermission, EgaPermissionRequest, EgaUser, @@ -209,15 +209,15 @@ export const egaApiClient = async () => { /** * GET request to retrieve all currently release datasets released for a DAC * @param dacId DacAccessionId - * @returns ZodResultAccumulator + * @returns ZodResultAccumulator */ const getDatasetsForDac = async ( dacId: DacAccessionId, - ): Promise, GetDatasetsForDacFailure>> => { + ): Promise, GetDatasetsForDacFailure>> => { const url = urlJoin(DACS, dacId, DATASETS); try { const { data } = await apiAxiosClient.get(url); - const result = safeParseArray(Dataset, data); + const result = safeParseArray(EgaDataset, data); return success(result); } catch (err) { const errMessage = getErrorMessage(err, `Error retrieving datasets for DAC ${dacId}.`); diff --git a/src/jobs/ega/egaPermissionsReconciliation.ts b/src/jobs/ega/egaPermissionsReconciliation.ts index 42d50ac..096d594 100644 --- a/src/jobs/ega/egaPermissionsReconciliation.ts +++ b/src/jobs/ega/egaPermissionsReconciliation.ts @@ -24,9 +24,9 @@ import { processPermissionsForApprovedUsers, processPermissionsForDataset, } from './services/permissions'; -import { getUsers } from './services/users'; +import { getEgaUsers } from './services/users'; import { isSuccess } from './types/results'; -import { getApprovedUsers } from './utils'; +import { getDacoApprovedUsers } from './utils'; const JOB_NAME = 'RECONCILE_EGA_PERMISSIONS'; @@ -40,7 +40,7 @@ const JOB_NAME = 'RECONCILE_EGA_PERMISSIONS'; */ async function runEgaPermissionsReconciliation() { // retrieve approved users list from daco system - const dacoUsers = await getApprovedUsers(); + const dacoUsers = await getDacoApprovedUsers(); // initialize EGA Axios client const egaClient = await egaApiClient(); @@ -57,7 +57,7 @@ async function runEgaPermissionsReconciliation() { } logger.debug(`Successfully retrieved ${datasets.data.success.length} for DAC ${dacId}.`); // retrieve corresponding users in EGA system - const egaUsers = await getUsers(egaClient, dacoUsers); + const egaUsers = await getEgaUsers(egaClient, dacoUsers); logger.debug(`Retrieved ${Object.keys(egaUsers).length} corresponding users from EGA.`); const datasetsRetrieved = datasets.data.success; logger.debug(`Retrieved ${datasetsRetrieved.length} datasets for ${dacId}.`); diff --git a/src/jobs/ega/services/permissions.ts b/src/jobs/ega/services/permissions.ts index 1fcf18f..5014d4a 100644 --- a/src/jobs/ega/services/permissions.ts +++ b/src/jobs/ega/services/permissions.ts @@ -23,7 +23,7 @@ import { EgaClient } from '../egaClient'; import { DatasetAccessionId } from '../types/common'; import { DEFAULT_LIMIT, DEFAULT_OFFSET, EGA_MAX_REQUEST_SIZE } from '../types/constants'; import { PermissionRequest, RevokePermission } from '../types/requests'; -import { Dataset, EgaDacoUser, EgaDacoUserMap } from '../types/responses'; +import { EgaDataset, EgaDacoUser, EgaDacoUserMap } from '../types/responses'; import { isSuccess } from '../types/results'; import { createPermissionApprovalRequest, @@ -94,7 +94,7 @@ export const createRequiredPermissions = async ( export const processPermissionsForApprovedUsers = async ( egaClient: EgaClient, egaUsers: EgaDacoUserMap, - datasets: Dataset[], + datasets: EgaDataset[], ) => { const userList = Object.values(egaUsers); for await (const approvedUser of userList) { @@ -105,17 +105,22 @@ export const processPermissionsForApprovedUsers = async ( approvedUser.id, dataset.accession_id, ); - if (isSuccess(existingPermission)) { - if (existingPermission.data.success.length === 0) { - // create permission request, add to requestList - const permissionRequest = createPermissionRequest( - approvedUser.username, - dataset.accession_id, - ); - permissionRequests.push(permissionRequest); - } - } else { - logger.info(`Error fetching existing permission: ${existingPermission.message}`); + switch (existingPermission.status) { + case 'SUCCESS': + // if no permissions exist for a DACO approved user, a permission request needs to be created + if (existingPermission.data.success.length === 0) { + // create permission request, add to requestList + const permissionRequest = createPermissionRequest( + approvedUser.username, + dataset.accession_id, + ); + permissionRequests.push(permissionRequest); + } + break; + case 'SERVER_ERROR': + default: + logger.info(`Error fetching existing permission: ${existingPermission.message}`); + break; } } if (permissionRequests.length) { @@ -180,7 +185,7 @@ export const processPermissionsForDataset = async ( } const setSize = permissionsSet.size; if (setSize > 0) { - logger.debug(`There are ${permissionsSet.size} permissions to remove.`); + logger.debug(`There are ${setSize} permissions to remove.`); permissionsSet.forEach((perm) => { const revokeReq = createRevokePermissionRequest(perm); permissionsToRevoke.push(revokeReq); diff --git a/src/jobs/ega/services/users.ts b/src/jobs/ega/services/users.ts index 2e48dbe..5d41b4a 100644 --- a/src/jobs/ega/services/users.ts +++ b/src/jobs/ega/services/users.ts @@ -23,7 +23,7 @@ import { EgaDacoUserMap } from '../types/responses'; import { ApprovedUser } from '../utils'; /** - * Retrieve EGA user data for each user on DACO approved list + * Retrieve corresponding EGA user data for each user on DACO Approved Users list * @param client EgaClient * @param dacoUsers ApprovedUser[] * @returns EgaDacoUserMap @@ -41,7 +41,7 @@ import { ApprovedUser } from '../utils'; * } * getUsers(client, approvedUsersList) */ -export const getUsers = async ( +export const getEgaUsers = async ( client: EgaClient, approvedUsers: ApprovedUser[], ): Promise => { diff --git a/src/jobs/ega/types/responses.ts b/src/jobs/ega/types/responses.ts index 35bf81c..aa904a0 100644 --- a/src/jobs/ega/types/responses.ts +++ b/src/jobs/ega/types/responses.ts @@ -50,12 +50,12 @@ export const Dac = z.object({ }); export type Dac = z.infer; -export const Dataset = z.object({ +export const EgaDataset = z.object({ accession_id: DatasetAccessionId, title: z.string().nullable(), // TODO: verify this is expected description: z.string().optional(), }); -export type Dataset = z.infer; +export type EgaDataset = z.infer; export const EgaUser = z.object({ id: z.number(), diff --git a/src/jobs/ega/utils.ts b/src/jobs/ega/utils.ts index 3093499..fa39cdf 100644 --- a/src/jobs/ega/utils.ts +++ b/src/jobs/ega/utils.ts @@ -54,10 +54,10 @@ const parseApprovedUsersForApplication = ( }; /** - * Retrieves applicant and collaborator information from all currently approved applications + * Retrieves applicant and collaborator information from all currently approved applications in the DAC-API db * @returns Promise */ -export const getApprovedUsers = async () => { +export const getDacoApprovedUsers = async () => { const results = await getUsersFromApprovedApps(); const parsedUsers = results.map((app) => parseApprovedUsersForApplication(app)).flat(); return uniqBy(parsedUsers, 'email'); From 81d23b6b371d8c94d9be7acd162b8425a651c3c8 Mon Sep 17 00:00:00 2001 From: Ann Catton Date: Fri, 11 Oct 2024 12:22:57 -0400 Subject: [PATCH 25/25] remove unneeded fields from EgaPermissionRequest response schema --- src/jobs/ega/types/common.ts | 4 ---- src/jobs/ega/types/responses.ts | 18 +----------------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/src/jobs/ega/types/common.ts b/src/jobs/ega/types/common.ts index 4c71527..818b659 100644 --- a/src/jobs/ega/types/common.ts +++ b/src/jobs/ega/types/common.ts @@ -23,10 +23,6 @@ import { z } from 'zod'; Dates * ******************* */ -// For YYYY-MM-DD Date strings (i.e. '2021-01-01') -export const DateString = z.string().date(); -export type DateString = z.infer; - // For ISO8601 Datetime strings (i.e. '2021-01-01T00:00:00.000Z') // Note: for safeParse to allow the +00:00, as in '2024-01-31T16:25:13.725724+00:00', would need .datetime({ offset: true }) export const DateTime = z.string().datetime({ offset: true }); diff --git a/src/jobs/ega/types/responses.ts b/src/jobs/ega/types/responses.ts index aa904a0..7e0712c 100644 --- a/src/jobs/ega/types/responses.ts +++ b/src/jobs/ega/types/responses.ts @@ -22,8 +22,6 @@ import { DacAccessionId, DacStatus, DatasetAccessionId, - DateString, - DateTime, IdpTokenType, UserAccessionId, } from './common'; @@ -66,23 +64,9 @@ export const EgaUser = z.object({ }); export type EgaUser = z.infer; +// the full response from EGA has several other fields, but we only parse for the request_id field required for the permission approval step in createRequiredPermissions() export const EgaPermissionRequest = z.object({ request_id: z.number(), - status: z.string(), - request_data: z.object({ - comment: z.string(), - }), - // TODO: api docs state this should be a DateTime string, but receiving 'YYYY-MM-DD` string. May need to change to coerceable date? - date: DateString, - username: z.string(), - full_name: z.string().nullable(), - email: z.string().email().nullable(), - organisation: z.string().nullable(), - dataset_accession_id: DatasetAccessionId, - dataset_title: z.string().nullable(), - dac_accession_id: DacAccessionId, - dac_comment: z.string().nullable(), - dac_comment_edited_at: DateTime.nullable(), // TODO: api docs state this should be DateTime string, but need to verify }); export type EgaPermissionRequest = z.infer;