Test new scrape method. Add scrape test to testFunction

game-ci · Jun 5, 2024 · e36703a · e36703a
1 parent 635bf34
commit e36703a
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 21 deletions.
diff --git a/functions/src/api/testFunction.ts b/functions/src/api/testFunction.ts
@@ -2,6 +2,8 @@ import { onRequest, Request } from 'firebase-functions/v2/https';
 import { Response } from 'express-serve-static-core';
 import { defineSecret } from 'firebase-functions/params';
 import { scrapeVersions } from '../logic/ingestRepoVersions/scrapeVersions';
+import { scrapeVersions as scrapeUnityVersions } from '../logic/ingestUnityVersions/scrapeVersions';
+
 import { Discord } from '../service/discord';
 
 const discordToken = defineSecret('DISCORD_TOKEN');
@@ -34,9 +36,15 @@ export const testFunction = onRequest(
       );
 
       if (versions.length === 0) {
-        info = 'No versions were found.';
-        code = 500;
+        throw new Error('No versions were found.');
+      }
+
+      const unityVersions = await scrapeUnityVersions();
+      if (unityVersions.length === 0) {
+        throw new Error('No Unity versions were found.');
       }
+
+      info = `Found ${versions.length} repo versions and ${unityVersions.length} Unity versions. First Unity Version: ${unityVersions[0].version}, ${unityVersions[0].changeSet}`;
     } catch (error: any) {
       info = error.message;
       code = 500;

diff --git a/functions/src/logic/ingestUnityVersions/scrapeVersions.ts b/functions/src/logic/ingestUnityVersions/scrapeVersions.ts
@@ -2,29 +2,30 @@ import { getDocumentFromUrl } from '../utils/get-document-from-url';
 import { EditorVersionInfo } from '../../model/editorVersionInfo';
 
 const UNITY_ARCHIVE_URL = 'https://unity.com/releases/editor/archive';
+const unity_version_regex = /unityhub:\/\/(\d+)\.(\d+)\.(\d+[a-zA-Z]\d+)\/(\w+)/g;
 
-/**
- * Based on https://github.com/BLaZeKiLL/unity-scraper
- */
 export const scrapeVersions = async (): Promise<EditorVersionInfo[]> => {
   const document = await getDocumentFromUrl(UNITY_ARCHIVE_URL);
 
-  const links = Array.from(document.querySelectorAll('.release-links div:first-child a[href]'));
-  const hrefs = links.map((a) => a.getAttribute('href')) as string[];
+  const scripts = document.querySelectorAll('script');
 
-  const versionInfoList = hrefs.map((href) => {
-    const info = href.replace('unityhub://', '');
-    const [version, changeSet] = info.split('/');
-    const [major, minor, patch] = version.split('.');
+  for (const script of scripts) {
+    if (script.textContent) {
+      const matches = [...script.textContent.matchAll(unity_version_regex)];
+      if (matches.length > 0) {
+        return matches.map((match) => {
+          const [_, major, minor, patch, changeSet] = match;
+          return {
+            version: `${major}.${minor}.${patch}`,
+            major: Number(major),
+            minor: Number(minor),
+            patch,
+            changeSet,
+          };
+        });
+      }
+    }
+  }
 
-    return {
-      version,
-      changeSet,
-      major: Number(major),
-      minor: Number(minor),
-      patch,
-    };
-  });
-
-  return versionInfoList;
+  throw new Error('No Unity versions found!');
 };