From 9412fa7efa32519bf635591d9714283d18dac9c4 Mon Sep 17 00:00:00 2001 From: Abdulrhmn Ghanem Date: Fri, 30 Jun 2023 15:42:22 +0300 Subject: [PATCH 1/3] Handle registry boards in processor --- processor/package.json | 1 + processor/src/queues.ts | 43 +++++++++++++++++++++ processor/test/integration/projects.test.ts | 6 +-- processor/yarn.lock | 39 +++++++++++++++++++ scripts/importBoardsTxt.ts | 11 +----- 5 files changed, 87 insertions(+), 13 deletions(-) diff --git a/processor/package.json b/processor/package.json index 6de27b7629..daef3b1fa2 100644 --- a/processor/package.json +++ b/processor/package.json @@ -30,6 +30,7 @@ "js-yaml": "^4.1.0", "jszip": "^3.8.0", "meilisearch": "^0.25.1", + "node-fetch": "^3.3.1", "pcb-stackup": "^4.2.5", "postgres": "^3.2.2", "prexit": "^1.0.0", diff --git a/processor/src/queues.ts b/processor/src/queues.ts index 41d6220110..c839ec2ce1 100644 --- a/processor/src/queues.ts +++ b/processor/src/queues.ts @@ -1,6 +1,7 @@ import AsyncLock from 'async-lock' import bullmq, { FlowProducer } from 'bullmq' import fs from 'node:fs/promises' +import fetch from 'node-fetch' import path from 'node:path' import { DATA_DIR, PROCESSOR_ASSET_VERSION } from './env.js' import { exists } from './utils.js' @@ -180,7 +181,14 @@ async function sync(gitDir, checkoutDir) { .acquire(gitDir, async done => { log.info('Acquired sync lock for ', gitDir) + const registryHash = getRegistryHash(gitDir) + if (await exists(checkoutDir)) { + if (registryHash) { + // no need to pull if we aren't going to use the latest commit + done() + } + log.debug('Pulling updates for', gitDir) await sh`cd ${checkoutDir} && git pull`.catch(err => { // repos with no branches yet will create this error @@ -201,6 +209,15 @@ async function sync(gitDir, checkoutDir) { } }) log.debug('Cloned into', checkoutDir) + if (registryHash) { + await sh`cd ${checkoutDir} && git checkout ${registryHash}` + .then(() => log.debug(`reset ${gitDir} to ${registryHash}`)) + .catch(err => { + log.error( + new Error(`failed to reset ${gitDir} to ${registryHash}: ${err}`), + ) + }) + } } done() }) @@ -234,3 +251,29 @@ async function alreadyProcessed( const allReportsExist = reports.every(r => r) return allReportsExist } + +async function getRegistryBoards() { + const url = + 'https://raw.githubusercontent.com/kitspace/kitspace/master/registry.json' + const response = await fetch(url) + const boards = (await response.json()) as Array + return boards +} + +const registryBoards = await getRegistryBoards() + +function getRegistryHash(localGitDir: string) { + const repoFullName = localGitDir + .split('/') + .slice(-2) + .join('/') + .replace('.git', '') + const registryBoard = registryBoards.find(b => b.repo.includes(repoFullName)) + + return registryBoard ? registryBoard.hash : null +} + +interface RegistryBoard { + repo: string + hash: string +} diff --git a/processor/test/integration/projects.test.ts b/processor/test/integration/projects.test.ts index 4e4c351079..5d6123f205 100644 --- a/processor/test/integration/projects.test.ts +++ b/processor/test/integration/projects.test.ts @@ -115,7 +115,7 @@ describe( id: '1', is_mirror: true, is_empty: false, - owner_name: 'kitspace', + owner_name: 'kitspace-test', default_branch: 'master', original_url: 'https://github.com/kitspace/ruler', name: 'ruler', @@ -130,7 +130,7 @@ describe( await sh`cd ${tmpBare} && git update-ref HEAD ${hash}` await sh`git clone --bare ${tmpBare} ${path.join( repoDir, - 'kitspace/ruler.git', + 'kitspace-test/ruler.git', )}` await dispatchRepoEvent(giteaDB, repoInfo, repoEventFixture) @@ -148,7 +148,7 @@ describe( const p = path.join( DATA_DIR, PROCESSOR_ASSET_VERSION, - `kitspace/ruler/${hash}/${f}`, + `kitspace-test/ruler/${hash}/${f}`, ) expect(s3.exists).toHaveBeenCalledWith(p) expect(s3.uploadFileContents).toHaveBeenCalledWith( diff --git a/processor/yarn.lock b/processor/yarn.lock index cb11e848af..d50f01220e 100644 --- a/processor/yarn.lock +++ b/processor/yarn.lock @@ -1851,6 +1851,11 @@ css-what@^5.0.0, css-what@^5.0.1: resolved "https://registry.yarnpkg.com/css-what/-/css-what-5.0.1.tgz#3efa820131f4669a8ac2408f9c32e7c7de9f4cad" integrity sha512-FYDTSHb/7KXsWICVsxdmiExPjCfRC4qRFBdVwv7Ax9hMnvMmEjP9RfxTEZ3qPZGmADDn2vAKSo9UcN1jKVYscg== +data-uri-to-buffer@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz#d8feb2b2881e6a4f58c2e08acfd0e2834e26222e" + integrity sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A== + debug@^2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -2484,6 +2489,14 @@ fault@^2.0.0: dependencies: format "^0.2.0" +fetch-blob@^3.1.2, fetch-blob@^3.1.4: + version "3.2.0" + resolved "https://registry.yarnpkg.com/fetch-blob/-/fetch-blob-3.2.0.tgz#f09b8d4bbd45adc6f0c20b7e787e793e309dcce9" + integrity sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ== + dependencies: + node-domexception "^1.0.0" + web-streams-polyfill "^3.0.3" + file-entry-cache@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" @@ -2532,6 +2545,13 @@ format@^0.2.0: resolved "https://registry.yarnpkg.com/format/-/format-0.2.2.tgz#d6170107e9efdc4ed30c9dc39016df942b5cb58b" integrity sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww== +formdata-polyfill@^4.0.10: + version "4.0.10" + resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423" + integrity sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g== + dependencies: + fetch-blob "^3.1.2" + formidable@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.2.tgz#bf69aea2972982675f00865342b982986f6b8dd9" @@ -3833,6 +3853,11 @@ node-cleanup@^2.1.2: resolved "https://registry.yarnpkg.com/node-cleanup/-/node-cleanup-2.1.2.tgz#7ac19abd297e09a7f72a71545d951b517e4dde2c" integrity sha1-esGavSl+Caf3KnFUXZUbUX5N3iw= +node-domexception@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" + integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== + node-emoji@^1.11.0: version "1.11.0" resolved "https://registry.yarnpkg.com/node-emoji/-/node-emoji-1.11.0.tgz#69a0150e6946e2f115e9d7ea4df7971e2628301c" @@ -3847,6 +3872,15 @@ node-fetch@2.6.7: dependencies: whatwg-url "^5.0.0" +node-fetch@^3.3.1: + version "3.3.1" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.3.1.tgz#b3eea7b54b3a48020e46f4f88b9c5a7430d20b2e" + integrity sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow== + dependencies: + data-uri-to-buffer "^4.0.0" + fetch-blob "^3.1.4" + formdata-polyfill "^4.0.10" + node-gyp-build-optional-packages@5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17" @@ -4931,6 +4965,11 @@ web-namespaces@^2.0.0: resolved "https://registry.yarnpkg.com/web-namespaces/-/web-namespaces-2.0.1.tgz#1010ff7c650eccb2592cebeeaf9a1b253fd40692" integrity sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ== +web-streams-polyfill@^3.0.3: + version "3.2.1" + resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz#71c2718c52b45fd49dbeee88634b3a60ceab42a6" + integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q== + webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" diff --git a/scripts/importBoardsTxt.ts b/scripts/importBoardsTxt.ts index 2aad18e6e6..5c433eaabb 100755 --- a/scripts/importBoardsTxt.ts +++ b/scripts/importBoardsTxt.ts @@ -45,12 +45,10 @@ const headers = { } async function main() { - const registryBoards = await getRegistryBoards() let boards = (await getBoardsTxt()) - .filter(url => !registryBoards.includes(url)) - .slice(0, flags.numberOfRepos) if (flags.shuffle) { boards = shuffle(boards) + .slice(0, flags.numberOfRepos) } const GithubReposDescriptions = await getAllGithubReposDescriptions(boards) @@ -74,13 +72,6 @@ async function getBoardsTxt() { return text.split('\n').filter(Boolean) } -async function getRegistryBoards() { - const url = - 'https://raw.githubusercontent.com/kitspace/kitspace/master/registry.json' - const response = await fetch(url) - const boards = (await response.json()) as RegistryBoard[] - return boards.map(board => board.repo) -} async function importRepo( url: string, From ba2707d225c6fdf35b6de72a33b930880ab40152 Mon Sep 17 00:00:00 2001 From: Ghanem <37152329+AbdulrhmnGhanem@users.noreply.github.com> Date: Mon, 3 Jul 2023 00:10:02 +0300 Subject: [PATCH 2/3] Apply suggestions from code review Co-authored-by: Kaspar Emanuel --- processor/src/queues.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/processor/src/queues.ts b/processor/src/queues.ts index c839ec2ce1..42713efd2e 100644 --- a/processor/src/queues.ts +++ b/processor/src/queues.ts @@ -184,7 +184,7 @@ async function sync(gitDir, checkoutDir) { const registryHash = getRegistryHash(gitDir) if (await exists(checkoutDir)) { - if (registryHash) { + if (registryHash != null) { // no need to pull if we aren't going to use the latest commit done() } @@ -209,7 +209,7 @@ async function sync(gitDir, checkoutDir) { } }) log.debug('Cloned into', checkoutDir) - if (registryHash) { + if (registryHash != null) { await sh`cd ${checkoutDir} && git checkout ${registryHash}` .then(() => log.debug(`reset ${gitDir} to ${registryHash}`)) .catch(err => { @@ -267,7 +267,7 @@ function getRegistryHash(localGitDir: string) { .split('/') .slice(-2) .join('/') - .replace('.git', '') + .replace(/\.git$/, '') const registryBoard = registryBoards.find(b => b.repo.includes(repoFullName)) return registryBoard ? registryBoard.hash : null From 68230b6960213a9696919c2986ef15b4a889148d Mon Sep 17 00:00:00 2001 From: Abdulrhmn Ghanem Date: Mon, 3 Jul 2023 00:24:05 +0300 Subject: [PATCH 3/3] fix early exit for registry cloning --- processor/src/queues.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/processor/src/queues.ts b/processor/src/queues.ts index 42713efd2e..50111b18f4 100644 --- a/processor/src/queues.ts +++ b/processor/src/queues.ts @@ -187,6 +187,7 @@ async function sync(gitDir, checkoutDir) { if (registryHash != null) { // no need to pull if we aren't going to use the latest commit done() + return } log.debug('Pulling updates for', gitDir)