Merge branch 'ag/import-registry' of https://github.com/kitspace/kits…

…pace-v2

processor/package.json

@@ -30,6 +30,7 @@
     "js-yaml": "^4.1.0",
     "jszip": "^3.8.0",
     "meilisearch": "^0.25.1",
+    "node-fetch": "^3.3.1",
     "pcb-stackup": "^4.2.5",
     "postgres": "^3.2.2",
     "prexit": "^1.0.0",

processor/src/queues.ts

@@ -1,6 +1,7 @@
 import AsyncLock from 'async-lock'
 import bullmq, { FlowProducer } from 'bullmq'
 import fs from 'node:fs/promises'
+import fetch from 'node-fetch'
 import path from 'node:path'
 import { DATA_DIR, PROCESSOR_ASSET_VERSION } from './env.js'
 import { exists } from './utils.js'
@@ -180,7 +181,15 @@ async function sync(gitDir, checkoutDir) {
     .acquire(gitDir, async done => {
       log.info('Acquired sync lock for ', gitDir)
 
+      const registryHash = getRegistryHash(gitDir)
+
       if (await exists(checkoutDir)) {
+        if (registryHash != null) {
+          // no need to pull if we aren't going to use the latest commit
+          done()
+          return
+        }
+
         log.debug('Pulling updates for', gitDir)
         await sh`cd ${checkoutDir} && git pull`.catch(err => {
           // repos with no branches yet will create this error
@@ -201,6 +210,15 @@ async function sync(gitDir, checkoutDir) {
           }
         })
         log.debug('Cloned into', checkoutDir)
+        if (registryHash != null) {
+          await sh`cd ${checkoutDir} && git checkout ${registryHash}`
+            .then(() => log.debug(`reset ${gitDir} to ${registryHash}`))
+            .catch(err => {
+              log.error(
+                new Error(`failed to reset ${gitDir} to ${registryHash}: ${err}`),
+              )
+            })
+        }
       }
       done()
     })
@@ -234,3 +252,29 @@ async function alreadyProcessed(
   const allReportsExist = reports.every(r => r)
   return allReportsExist
 }
+
+async function getRegistryBoards() {
+  const url =
+    'https://raw.githubusercontent.com/kitspace/kitspace/master/registry.json'
+  const response = await fetch(url)
+  const boards = (await response.json()) as Array<RegistryBoard>
+  return boards
+}
+
+const registryBoards = await getRegistryBoards()
+
+function getRegistryHash(localGitDir: string) {
+  const repoFullName = localGitDir
+    .split('/')
+    .slice(-2)
+    .join('/')
+    .replace(/\.git$/, '')
+  const registryBoard = registryBoards.find(b => b.repo.includes(repoFullName))
+
+  return registryBoard ? registryBoard.hash : null
+}
+
+interface RegistryBoard {
+  repo: string
+  hash: string
+}

processor/test/integration/projects.test.ts

@@ -115,7 +115,7 @@ describe(
           id: '1',
           is_mirror: true,
           is_empty: false,
-          owner_name: 'kitspace',
+          owner_name: 'kitspace-test',
           default_branch: 'master',
           original_url: 'https://github.com/kitspace/ruler',
           name: 'ruler',
@@ -130,7 +130,7 @@ describe(
         await sh`cd ${tmpBare} && git update-ref HEAD ${hash}`
         await sh`git clone --bare ${tmpBare} ${path.join(
           repoDir,
-          'kitspace/ruler.git',
+          'kitspace-test/ruler.git',
         )}`
 
         await dispatchRepoEvent(giteaDB, repoInfo, repoEventFixture)
@@ -148,7 +148,7 @@ describe(
           const p = path.join(
             DATA_DIR,
             PROCESSOR_ASSET_VERSION,
-            `kitspace/ruler/${hash}/${f}`,
+            `kitspace-test/ruler/${hash}/${f}`,
           )
           expect(s3.exists).toHaveBeenCalledWith(p)
           expect(s3.uploadFileContents).toHaveBeenCalledWith(

processor/yarn.lock

@@ -1851,6 +1851,11 @@ css-what@^5.0.0, css-what@^5.0.1:
   resolved "https://registry.yarnpkg.com/css-what/-/css-what-5.0.1.tgz#3efa820131f4669a8ac2408f9c32e7c7de9f4cad"
   integrity sha512-FYDTSHb/7KXsWICVsxdmiExPjCfRC4qRFBdVwv7Ax9hMnvMmEjP9RfxTEZ3qPZGmADDn2vAKSo9UcN1jKVYscg==
 
+data-uri-to-buffer@^4.0.0:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz#d8feb2b2881e6a4f58c2e08acfd0e2834e26222e"
+  integrity sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==
+
 debug@^2.6.9:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -2484,6 +2489,14 @@ fault@^2.0.0:
   dependencies:
     format "^0.2.0"
 
+fetch-blob@^3.1.2, fetch-blob@^3.1.4:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/fetch-blob/-/fetch-blob-3.2.0.tgz#f09b8d4bbd45adc6f0c20b7e787e793e309dcce9"
+  integrity sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==
+  dependencies:
+    node-domexception "^1.0.0"
+    web-streams-polyfill "^3.0.3"
+
 file-entry-cache@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027"
@@ -2532,6 +2545,13 @@ format@^0.2.0:
   resolved "https://registry.yarnpkg.com/format/-/format-0.2.2.tgz#d6170107e9efdc4ed30c9dc39016df942b5cb58b"
   integrity sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==
 
+formdata-polyfill@^4.0.10:
+  version "4.0.10"
+  resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423"
+  integrity sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==
+  dependencies:
+    fetch-blob "^3.1.2"
+
 formidable@^1.2.2:
   version "1.2.2"
   resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.2.tgz#bf69aea2972982675f00865342b982986f6b8dd9"
@@ -3833,6 +3853,11 @@ node-cleanup@^2.1.2:
   resolved "https://registry.yarnpkg.com/node-cleanup/-/node-cleanup-2.1.2.tgz#7ac19abd297e09a7f72a71545d951b517e4dde2c"
   integrity sha1-esGavSl+Caf3KnFUXZUbUX5N3iw=
 
+node-domexception@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
+  integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
+
 node-emoji@^1.11.0:
   version "1.11.0"
   resolved "https://registry.yarnpkg.com/node-emoji/-/node-emoji-1.11.0.tgz#69a0150e6946e2f115e9d7ea4df7971e2628301c"
@@ -3847,6 +3872,15 @@ node-fetch@2.6.7:
   dependencies:
     whatwg-url "^5.0.0"
 
+node-fetch@^3.3.1:
+  version "3.3.1"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.3.1.tgz#b3eea7b54b3a48020e46f4f88b9c5a7430d20b2e"
+  integrity sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow==
+  dependencies:
+    data-uri-to-buffer "^4.0.0"
+    fetch-blob "^3.1.4"
+    formdata-polyfill "^4.0.10"
+
 node-gyp-build-optional-packages@5.0.3:
   version "5.0.3"
   resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17"
@@ -4931,6 +4965,11 @@ web-namespaces@^2.0.0:
   resolved "https://registry.yarnpkg.com/web-namespaces/-/web-namespaces-2.0.1.tgz#1010ff7c650eccb2592cebeeaf9a1b253fd40692"
   integrity sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==
 
+web-streams-polyfill@^3.0.3:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz#71c2718c52b45fd49dbeee88634b3a60ceab42a6"
+  integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==
+
 webidl-conversions@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"

scripts/importBoardsTxt.ts

@@ -45,12 +45,10 @@ const headers = {
 }
 
 async function main() {
-  const registryBoards = await getRegistryBoards()
   let boards = (await getBoardsTxt())
-    .filter(url => !registryBoards.includes(url))
-    .slice(0, flags.numberOfRepos)
   if (flags.shuffle) {
     boards = shuffle(boards)
+      .slice(0, flags.numberOfRepos)
   }
   const GithubReposDescriptions = await getAllGithubReposDescriptions(boards)
 
@@ -74,13 +72,6 @@ async function getBoardsTxt() {
   return text.split('\n').filter(Boolean)
 }
 
-async function getRegistryBoards() {
-  const url =
-    'https://raw.githubusercontent.com/kitspace/kitspace/master/registry.json'
-  const response = await fetch(url)
-  const boards = (await response.json()) as RegistryBoard[]
-  return boards.map(board => board.repo)
-}
 
 async function importRepo(
   url: string,