From dbb9b53a2680609ca8c6f2113ba1bc3dbd845f95 Mon Sep 17 00:00:00 2001 From: Branden Rodgers Date: Fri, 1 Dec 2023 16:17:06 -0500 Subject: [PATCH 1/4] separating github api calls into new file. Adding some basic tests --- api/github.ts | 70 ++++++++++++++++++++++++++++++ lang/en.json | 2 +- lib/__tests__/github.ts | 44 +++++++++++++++++++ lib/__tests__/trackUsage.ts | 75 ++++++++++++++++++++++++++++++++ lib/{ => cms}/validate.ts | 10 ++--- lib/github.ts | 86 +++++++++++++++---------------------- 6 files changed, 229 insertions(+), 58 deletions(-) create mode 100644 api/github.ts create mode 100644 lib/__tests__/github.ts create mode 100644 lib/__tests__/trackUsage.ts rename lib/{ => cms}/validate.ts (80%) diff --git a/api/github.ts b/api/github.ts new file mode 100644 index 00000000..56ad1306 --- /dev/null +++ b/api/github.ts @@ -0,0 +1,70 @@ +import axios, { AxiosResponse } from 'axios'; +import { DEFAULT_USER_AGENT_HEADERS } from '../http/getAxiosConfig'; +import { GithubReleaseData, GithubRepoFile } from '../types/Github'; + +const GITHUB_REPOS_API = 'https://api.github.com/repos'; +export const GITHUB_RAW_CONTENT_API_PATH = 'https://raw.githubusercontent.com'; + +declare global { + // eslint-disable-next-line no-var + var githubToken: string; +} + +type RepoPath = `${string}/${string}`; + +const GITHUB_AUTH_HEADERS = { + authorization: + global && global.githubToken ? `Bearer ${global.githubToken}` : null, +}; + +// Returns information about the repo's releases. Defaults to "latest" if no tag is provided +// https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#get-a-release-by-tag-name +export async function fetchRepoReleaseData( + repoPath: RepoPath, + tag = '' +): Promise> { + const URL = `${GITHUB_REPOS_API}/${repoPath}/releases`; + + return axios.get( + `${URL}/${tag ? `tags/${tag}` : 'latest'}`, + { + headers: { ...DEFAULT_USER_AGENT_HEADERS, ...GITHUB_AUTH_HEADERS }, + } + ); +} + +// Returns the entire repo content as a zip, using the zipball_url from fetchRepoReleaseData() +// https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#download-a-repository-archive-zip +export async function fetchRepoAsZip( + zipUrl: string +): Promise> { + return axios.get(zipUrl, { + headers: { ...DEFAULT_USER_AGENT_HEADERS, ...GITHUB_AUTH_HEADERS }, + }); +} + +// Returns the raw file contents via the raw.githubusercontent endpoint +export async function fetchRepoFile( + downloadUrl: string +): Promise> { + return axios.get(downloadUrl, { + headers: { ...DEFAULT_USER_AGENT_HEADERS, ...GITHUB_AUTH_HEADERS }, + }); +} + +// Returns the contents of a file or directory in a repository by path +// https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#get-repository-content +export async function fetchRepoContents( + repoPath: RepoPath, + path: string, + ref?: string +): Promise>> { + const refQuery = ref ? `?ref=${ref}` : ''; + + return axios.get>( + `${GITHUB_REPOS_API}/${repoPath}/contents/${path}${refQuery}`, + { + headers: { ...DEFAULT_USER_AGENT_HEADERS, ...GITHUB_AUTH_HEADERS }, + } + ); +} diff --git a/lang/en.json b/lang/en.json index d0e60bdc..5b71579a 100644 --- a/lang/en.json +++ b/lang/en.json @@ -30,7 +30,7 @@ } }, "github": { - "fetchJsonFromRepository": { + "fetchFileFromRepository": { "fetching": "Fetching {{ url }}...", "errors": { "fetchFail": "An error occured fetching JSON file." diff --git a/lib/__tests__/github.ts b/lib/__tests__/github.ts new file mode 100644 index 00000000..c763e95b --- /dev/null +++ b/lib/__tests__/github.ts @@ -0,0 +1,44 @@ +import { fetchFileFromRepository } from '../github'; +import { + GITHUB_RAW_CONTENT_API_PATH, + fetchRepoFile as __fetchRepoFile, +} from '../../api/github'; + +jest.mock('../../api/github'); + +const fetchRepoFile = __fetchRepoFile as jest.MockedFunction< + typeof __fetchRepoFile +>; + +describe('lib/github', () => { + describe('fetchFileFromRepository()', () => { + beforeAll(() => { + fetchRepoFile.mockResolvedValue({ data: null }); + }); + + afterAll(() => { + fetchRepoFile.mockReset(); + }); + + it('downloads a github repo and writes it to a destination folder', async () => { + await fetchFileFromRepository('owner/repo', 'file', 'ref'); + expect(fetchRepoFile).toHaveBeenCalledWith( + `${GITHUB_RAW_CONTENT_API_PATH}/owner/repo/ref/file` + ); + }); + }); + + // describe('fetchReleaseData()', () => { + // it('downloads a github repo and writes it to a destination folder', async () => { + // await cloneGithubRepo('./', 'test', 'github.com/repo', '', {}); + // expect(true).toBe(true); + // }); + // }); + + // describe('cloneGithubRepo()', () => { + // it('downloads a github repo and writes it to a destination folder', async () => { + // await cloneGithubRepo('./', 'test', 'github.com/repo', '', {}); + // expect(true).toBe(true); + // }); + // }); +}); diff --git a/lib/__tests__/trackUsage.ts b/lib/__tests__/trackUsage.ts new file mode 100644 index 00000000..3861f939 --- /dev/null +++ b/lib/__tests__/trackUsage.ts @@ -0,0 +1,75 @@ +import axios from 'axios'; +import { trackUsage } from '../trackUsage'; +import { + getAccountConfig as __getAccountConfig, + getAndLoadConfigIfNeeded as __getAndLoadConfigIfNeeded, +} from '../../config'; +import { AuthType } from '../../types/Accounts'; +import { ENVIRONMENTS } from '../../constants/environments'; + +jest.mock('axios'); +jest.mock('../../config'); + +const mockedAxios = jest.mocked(axios); +const getAccountConfig = __getAccountConfig as jest.MockedFunction< + typeof __getAccountConfig +>; +const getAndLoadConfigIfNeeded = + __getAndLoadConfigIfNeeded as jest.MockedFunction< + typeof __getAndLoadConfigIfNeeded + >; + +mockedAxios.mockResolvedValue({}); +getAndLoadConfigIfNeeded.mockReturnValue({}); + +const account = { + accountId: 12345, + authType: 'personalaccesskey' as AuthType, + personalAccessKey: 'let-me-in-3', + auth: { + tokenInfo: { + expiresAt: '', + accessToken: 'test-token', + }, + }, + env: ENVIRONMENTS.QA, +}; + +const usageTrackingMeta = { + action: 'cli-command', + command: 'test-command', +}; + +describe('lib/trackUsage', () => { + describe('trackUsage()', () => { + beforeEach(() => { + mockedAxios.mockClear(); + getAccountConfig.mockReset(); + getAccountConfig.mockReturnValue(account); + }); + + it('tracks correctly for unauthenticated accounts', async () => { + await trackUsage('test-action', 'INTERACTION', usageTrackingMeta); + const requestArgs = mockedAxios.mock.lastCall + ? mockedAxios.mock.lastCall[0] + : ({} as any); // eslint-disable-line @typescript-eslint/no-explicit-any + + expect(mockedAxios).toHaveBeenCalled(); + expect(requestArgs!.data.eventName).toEqual('test-action'); + expect(requestArgs!.url.includes('authenticated')).toBeFalsy(); + expect(getAccountConfig).not.toHaveBeenCalled(); + }); + + it('tracks correctly for authenticated accounts', async () => { + await trackUsage('test-action', 'INTERACTION', usageTrackingMeta, 12345); + const requestArgs = mockedAxios.mock.lastCall + ? mockedAxios.mock.lastCall[0] + : ({} as any); // eslint-disable-line @typescript-eslint/no-explicit-any + + expect(mockedAxios).toHaveBeenCalled(); + expect(requestArgs!.data.eventName).toEqual('test-action'); + expect(requestArgs!.url.includes('authenticated')).toBeTruthy(); + expect(getAccountConfig).toHaveBeenCalled(); + }); + }); +}); diff --git a/lib/validate.ts b/lib/cms/validate.ts similarity index 80% rename from lib/validate.ts rename to lib/cms/validate.ts index 98f791b6..2ea48a2f 100644 --- a/lib/validate.ts +++ b/lib/cms/validate.ts @@ -1,9 +1,9 @@ import fs from 'fs-extra'; -import { HUBL_EXTENSIONS } from '../constants/extensions'; -import { validateHubl } from '../api/validateHubl'; -import { walk } from './fs'; -import { getExt } from './path'; -import { LintResult } from '../types/HublValidation'; +import { HUBL_EXTENSIONS } from '../../constants/extensions'; +import { validateHubl } from '../../api/validateHubl'; +import { walk } from '../fs'; +import { getExt } from '../path'; +import { LintResult } from '../../types/HublValidation'; export async function lint( accountId: number, diff --git a/lib/github.ts b/lib/github.ts index 35b2fe92..63fedcce 100644 --- a/lib/github.ts +++ b/lib/github.ts @@ -1,4 +1,3 @@ -import axios from 'axios'; import path from 'path'; import fs from 'fs-extra'; @@ -7,42 +6,36 @@ import { throwError, throwErrorWithMessage } from '../errors/standardErrors'; import { extractZipArchive } from './archive'; import { GITHUB_RELEASE_TYPES } from '../constants/github'; -import { DEFAULT_USER_AGENT_HEADERS } from '../http/getAxiosConfig'; import { BaseError } from '../types/Error'; import { GithubReleaseData, GithubRepoFile } from '../types/Github'; import { ValueOf } from '../types/Utils'; import { LogCallbacksArg } from '../types/LogCallbacks'; +import { + GITHUB_RAW_CONTENT_API_PATH, + fetchRepoFile, + fetchRepoAsZip, + fetchRepoReleaseData, + fetchRepoContents, +} from '../api/github'; const i18nKey = 'lib.github'; -declare global { - // eslint-disable-next-line no-var - var githubToken: string; -} - type RepoPath = `${string}/${string}`; -const GITHUB_AUTH_HEADERS = { - authorization: - global && global.githubToken ? `Bearer ${global.githubToken}` : null, -}; - -export async function fetchJsonFromRepository( +export async function fetchFileFromRepository( repoPath: RepoPath, filePath: string, ref: string -): Promise { +): Promise { try { - const URL = `https://raw.githubusercontent.com/${repoPath}/${ref}/${filePath}`; - debug(`${i18nKey}.fetchJsonFromRepository.fetching`, { url: URL }); + const contentPath = `${GITHUB_RAW_CONTENT_API_PATH}/${repoPath}/${ref}/${filePath}`; + debug(`${i18nKey}.fetchFileFromRepository.fetching`, { url: contentPath }); - const { data } = await axios.get(URL, { - headers: { ...DEFAULT_USER_AGENT_HEADERS, ...GITHUB_AUTH_HEADERS }, - }); + const { data } = await fetchRepoFile(contentPath); return data; } catch (err) { throwErrorWithMessage( - `${i18nKey}.fetchJsonFromRepository.errors.fetchFail`, + `${i18nKey}.fetchFileFromRepository.errors.fetchFail`, {}, err as BaseError ); @@ -57,13 +50,8 @@ export async function fetchReleaseData( if (tag.length && tag[0] !== 'v') { tag = `v${tag}`; } - const URI = tag - ? `https://api.github.com/repos/${repoPath}/releases/tags/${tag}` - : `https://api.github.com/repos/${repoPath}/releases/latest`; try { - const { data } = await axios.get(URI, { - headers: { ...DEFAULT_USER_AGENT_HEADERS, ...GITHUB_AUTH_HEADERS }, - }); + const { data } = await fetchRepoReleaseData(repoPath, tag); return data; } catch (err) { const error = err as BaseError; @@ -99,9 +87,7 @@ async function downloadGithubRepoZip( const { name } = releaseData; debug(`${i18nKey}.downloadGithubRepoZip.fetchingName`, { name }); } - const { data } = await axios.get(zipUrl, { - headers: { ...DEFAULT_USER_AGENT_HEADERS, ...GITHUB_AUTH_HEADERS }, - }); + const { data } = await fetchRepoAsZip(zipUrl); debug(`${i18nKey}.downloadGithubRepoZip.completed`); return data; } catch (err) { @@ -144,29 +130,11 @@ export async function cloneGithubRepo( return success; } -async function getGitHubRepoContentsAtPath( - repoPath: RepoPath, - path: string, - ref?: string -): Promise> { - const refQuery = ref ? `?ref=${ref}` : ''; - const contentsRequestUrl = `https://api.github.com/repos/${repoPath}/contents/${path}${refQuery}`; - - const response = await axios.get>(contentsRequestUrl, { - headers: { ...DEFAULT_USER_AGENT_HEADERS, ...GITHUB_AUTH_HEADERS }, - }); - - return response.data; -} - async function fetchGitHubRepoContentFromDownloadUrl( dest: string, downloadUrl: string ): Promise { - const resp = await axios.get(downloadUrl, { - headers: { ...DEFAULT_USER_AGENT_HEADERS, ...GITHUB_AUTH_HEADERS }, - }); - + const resp = await fetchRepoFile(downloadUrl); fs.writeFileSync(dest, resp.data, 'utf8'); } @@ -177,11 +145,11 @@ export async function downloadGithubRepoContents( dest: string, ref?: string, filter?: (contentPiecePath: string, downloadPath: string) => boolean -): Promise { +): Promise { fs.ensureDirSync(path.dirname(dest)); try { - const contentsResp = await getGitHubRepoContentsAtPath( + const { data: contentsResp } = await fetchRepoContents( repoPath, contentPath, ref @@ -190,7 +158,11 @@ export async function downloadGithubRepoContents( const downloadContent = async ( contentPiece: GithubRepoFile ): Promise => { - const { path: contentPiecePath, download_url } = contentPiece; + const { + path: contentPiecePath, + download_url, + type: contentPieceType, + } = contentPiece; const downloadPath = path.join( dest, contentPiecePath.replace(contentPath, '') @@ -206,6 +178,16 @@ export async function downloadGithubRepoContents( downloadPath, }); + if (contentPieceType === 'dir') { + const { data: innerDirContent } = await fetchRepoContents( + repoPath, + contentPiecePath, + ref + ); + await Promise.all(innerDirContent.map(downloadContent)); + return Promise.resolve(); + } + return fetchGitHubRepoContentFromDownloadUrl(downloadPath, download_url); }; @@ -217,7 +199,7 @@ export async function downloadGithubRepoContents( contentPromises = [downloadContent(contentsResp)]; } - Promise.all(contentPromises); + return Promise.all(contentPromises); } catch (e) { const error = e as BaseError; if (error?.error?.message) { From b728ac63edd426f9b36eee5f221c31e764c2af3f Mon Sep 17 00:00:00 2001 From: Branden Rodgers Date: Fri, 1 Dec 2023 16:33:33 -0500 Subject: [PATCH 2/4] fix ts issue, remove commented code --- lib/__tests__/github.ts | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/lib/__tests__/github.ts b/lib/__tests__/github.ts index c763e95b..433eb165 100644 --- a/lib/__tests__/github.ts +++ b/lib/__tests__/github.ts @@ -13,7 +13,8 @@ const fetchRepoFile = __fetchRepoFile as jest.MockedFunction< describe('lib/github', () => { describe('fetchFileFromRepository()', () => { beforeAll(() => { - fetchRepoFile.mockResolvedValue({ data: null }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fetchRepoFile.mockResolvedValue({ data: null } as any); }); afterAll(() => { @@ -27,18 +28,4 @@ describe('lib/github', () => { ); }); }); - - // describe('fetchReleaseData()', () => { - // it('downloads a github repo and writes it to a destination folder', async () => { - // await cloneGithubRepo('./', 'test', 'github.com/repo', '', {}); - // expect(true).toBe(true); - // }); - // }); - - // describe('cloneGithubRepo()', () => { - // it('downloads a github repo and writes it to a destination folder', async () => { - // await cloneGithubRepo('./', 'test', 'github.com/repo', '', {}); - // expect(true).toBe(true); - // }); - // }); }); From 18028bf60f3f5d8316cd0861bf4eb4c11fdf180b Mon Sep 17 00:00:00 2001 From: Branden Rodgers Date: Wed, 6 Dec 2023 16:48:53 -0500 Subject: [PATCH 3/4] small updates from review --- api/github.ts | 15 ++++++++++----- lang/en.json | 2 +- lib/github.ts | 12 ++++++------ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/api/github.ts b/api/github.ts index 56ad1306..230f9fb1 100644 --- a/api/github.ts +++ b/api/github.ts @@ -3,7 +3,7 @@ import { DEFAULT_USER_AGENT_HEADERS } from '../http/getAxiosConfig'; import { GithubReleaseData, GithubRepoFile } from '../types/Github'; const GITHUB_REPOS_API = 'https://api.github.com/repos'; -export const GITHUB_RAW_CONTENT_API_PATH = 'https://raw.githubusercontent.com'; +const GITHUB_RAW_CONTENT_API_PATH = 'https://raw.githubusercontent.com'; declare global { // eslint-disable-next-line no-var @@ -45,11 +45,16 @@ export async function fetchRepoAsZip( // Returns the raw file contents via the raw.githubusercontent endpoint export async function fetchRepoFile( - downloadUrl: string + repoPath: RepoPath, + filePath: string, + ref: string ): Promise> { - return axios.get(downloadUrl, { - headers: { ...DEFAULT_USER_AGENT_HEADERS, ...GITHUB_AUTH_HEADERS }, - }); + return axios.get( + `${GITHUB_RAW_CONTENT_API_PATH}/${repoPath}/${ref}/${filePath}`, + { + headers: { ...DEFAULT_USER_AGENT_HEADERS, ...GITHUB_AUTH_HEADERS }, + } + ); } // Returns the contents of a file or directory in a repository by path diff --git a/lang/en.json b/lang/en.json index 5b71579a..f7851639 100644 --- a/lang/en.json +++ b/lang/en.json @@ -31,7 +31,7 @@ }, "github": { "fetchFileFromRepository": { - "fetching": "Fetching {{ url }}...", + "fetching": "Fetching {{ path }}...", "errors": { "fetchFail": "An error occured fetching JSON file." } diff --git a/lib/github.ts b/lib/github.ts index 63fedcce..e392f605 100644 --- a/lib/github.ts +++ b/lib/github.ts @@ -11,7 +11,6 @@ import { GithubReleaseData, GithubRepoFile } from '../types/Github'; import { ValueOf } from '../types/Utils'; import { LogCallbacksArg } from '../types/LogCallbacks'; import { - GITHUB_RAW_CONTENT_API_PATH, fetchRepoFile, fetchRepoAsZip, fetchRepoReleaseData, @@ -28,10 +27,11 @@ export async function fetchFileFromRepository( ref: string ): Promise { try { - const contentPath = `${GITHUB_RAW_CONTENT_API_PATH}/${repoPath}/${ref}/${filePath}`; - debug(`${i18nKey}.fetchFileFromRepository.fetching`, { url: contentPath }); + debug(`${i18nKey}.fetchFileFromRepository.fetching`, { + path: `${repoPath}/${ref}/${filePath}`, + }); - const { data } = await fetchRepoFile(contentPath); + const { data } = await fetchRepoFile(repoPath, filePath, ref); return data; } catch (err) { throwErrorWithMessage( @@ -145,7 +145,7 @@ export async function downloadGithubRepoContents( dest: string, ref?: string, filter?: (contentPiecePath: string, downloadPath: string) => boolean -): Promise { +): Promise { fs.ensureDirSync(path.dirname(dest)); try { @@ -199,7 +199,7 @@ export async function downloadGithubRepoContents( contentPromises = [downloadContent(contentsResp)]; } - return Promise.all(contentPromises); + await Promise.all(contentPromises); } catch (e) { const error = e as BaseError; if (error?.error?.message) { From 7be2ecd7a0317f0f8a72ccd46785e4055722f3ed Mon Sep 17 00:00:00 2001 From: Branden Rodgers Date: Wed, 6 Dec 2023 17:03:15 -0500 Subject: [PATCH 4/4] fixing tests --- api/github.ts | 9 +++++++++ lib/__tests__/github.ts | 9 ++------- lib/github.ts | 3 ++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/api/github.ts b/api/github.ts index 230f9fb1..6584408f 100644 --- a/api/github.ts +++ b/api/github.ts @@ -57,6 +57,15 @@ export async function fetchRepoFile( ); } +// Returns the raw file contents via the raw.githubusercontent endpoint +export async function fetchRepoFileByDownloadUrl( + downloadUrl: string +): Promise> { + return axios.get(downloadUrl, { + headers: { ...DEFAULT_USER_AGENT_HEADERS, ...GITHUB_AUTH_HEADERS }, + }); +} + // Returns the contents of a file or directory in a repository by path // https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#get-repository-content export async function fetchRepoContents( diff --git a/lib/__tests__/github.ts b/lib/__tests__/github.ts index 433eb165..8b5c19af 100644 --- a/lib/__tests__/github.ts +++ b/lib/__tests__/github.ts @@ -1,8 +1,5 @@ import { fetchFileFromRepository } from '../github'; -import { - GITHUB_RAW_CONTENT_API_PATH, - fetchRepoFile as __fetchRepoFile, -} from '../../api/github'; +import { fetchRepoFile as __fetchRepoFile } from '../../api/github'; jest.mock('../../api/github'); @@ -23,9 +20,7 @@ describe('lib/github', () => { it('downloads a github repo and writes it to a destination folder', async () => { await fetchFileFromRepository('owner/repo', 'file', 'ref'); - expect(fetchRepoFile).toHaveBeenCalledWith( - `${GITHUB_RAW_CONTENT_API_PATH}/owner/repo/ref/file` - ); + expect(fetchRepoFile).toHaveBeenCalledWith('owner/repo', 'file', 'ref'); }); }); }); diff --git a/lib/github.ts b/lib/github.ts index e392f605..b17cc72c 100644 --- a/lib/github.ts +++ b/lib/github.ts @@ -12,6 +12,7 @@ import { ValueOf } from '../types/Utils'; import { LogCallbacksArg } from '../types/LogCallbacks'; import { fetchRepoFile, + fetchRepoFileByDownloadUrl, fetchRepoAsZip, fetchRepoReleaseData, fetchRepoContents, @@ -134,7 +135,7 @@ async function fetchGitHubRepoContentFromDownloadUrl( dest: string, downloadUrl: string ): Promise { - const resp = await fetchRepoFile(downloadUrl); + const resp = await fetchRepoFileByDownloadUrl(downloadUrl); fs.writeFileSync(dest, resp.data, 'utf8'); }