From 34b86740ee48f6521f35da87612a23564c3506c3 Mon Sep 17 00:00:00 2001 From: Eric Zhang Date: Mon, 23 Dec 2024 12:51:00 -0600 Subject: [PATCH] Speed up archiveStaleGames with populatedAt and concurrency --- functions/src/index.ts | 48 ++++++++++++++++++++----------- scripts/package-lock.json | 60 +++++++++++++++++++++++++++++++++++++-- scripts/package.json | 3 +- 3 files changed, 90 insertions(+), 21 deletions(-) diff --git a/functions/src/index.ts b/functions/src/index.ts index 700c651..d5a1b12 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -17,6 +17,10 @@ const stripe = process.env.FUNCTIONS_EMULATOR apiVersion: "2024-12-18.acacia", }); +// Hack: In Firebase v13, `admin.database.ServerValue.TIMESTAMP` +// does not work anymore from TypeScript. +const SERVER_VALUE_TIMESTAMP = { ".sv": "timestamp" }; + const MAX_GAME_ID_LENGTH = 64; const MAX_UNFINISHED_GAMES_PER_HOUR = 4; @@ -270,9 +274,7 @@ export const createGame = functions.https.onCall(async (data, context) => { } return { host: userId, - // Hack: In Firebase v13, `admin.database.ServerValue.TIMESTAMP` - // does not work anymore from TypeScript. - createdAt: { ".sv": "timestamp" }, + createdAt: SERVER_VALUE_TIMESTAMP, status: "waiting", access, mode, @@ -298,6 +300,7 @@ export const createGame = functions.https.onCall(async (data, context) => { updates.push( getDatabase().ref(`gameData/${gameId}`).set({ deck: generateDeck(), + populatedAt: SERVER_VALUE_TIMESTAMP, }), ); updates.push( @@ -425,8 +428,16 @@ export const handleStripe = functions.https.onRequest(async (req, res) => { * Archive a game state from RTDB to GCS, reducing the storage tier. * Returns whether the state was found in the database. */ -async function archiveGameState(gameId: string): Promise { - const snap = await getDatabase().ref(`gameData/${gameId}`).get(); +async function archiveGameState( + gameId: string, + snapInit?: DataSnapshot, +): Promise { + let snap: DataSnapshot; + if (snapInit) { + snap = snapInit; + } else { + snap = await getDatabase().ref(`gameData/${gameId}`).get(); + } if (!snap.exists()) { return false; // Game state is not present in the database, maybe racy? } @@ -458,7 +469,9 @@ async function restoreGameState(gameId: string): Promise { const jsonBlob = await gzip.decompress(gzippedBlob); const gameData = JSON.parse(jsonBlob.toString()); - await getDatabase().ref(`gameData/${gameId}`).set(gameData); + await getDatabase() + .ref(`gameData/${gameId}`) + .set({ ...gameData, populatedAt: SERVER_VALUE_TIMESTAMP }); return true; } @@ -493,20 +506,21 @@ export const fetchStaleGame = functions.https.onCall(async (data, context) => { /** Archive stale game states to GCS for cost savings. */ export const archiveStaleGames = functions - .runWith({ timeoutSeconds: 540, memory: "1GB" }) + .runWith({ timeoutSeconds: 540, memory: "2GB" }) .pubsub.schedule("every 1 hours") .onRun(async (context) => { - const cutoff = Date.now() - 30 * 86400 * 1000; // 30 days ago - const queue = new PQueue({ concurrency: 50 }); - - for await (const [gameId] of databaseIterator("gameData")) { - await queue.add(async () => { - const game = await getDatabase().ref(`games/${gameId}`).get(); - if (game.val().createdAt < cutoff) { + const cutoff = Date.now() - 14 * 86400 * 1000; // 14 days ago + const queue = new PQueue({ concurrency: 200 }); + + for await (const [gameId, gameState] of databaseIterator("gameData")) { + const populatedAt: number | null = gameState.child("populatedAt").val(); + if (!populatedAt || populatedAt < cutoff) { + await queue.onEmpty(); + queue.add(async () => { console.log(`Archiving stale game state for ${gameId}`); - await archiveGameState(gameId); - } - }); + await archiveGameState(gameId, gameState); + }); + } } await queue.onIdle(); diff --git a/scripts/package-lock.json b/scripts/package-lock.json index 12af7d6..854d74c 100644 --- a/scripts/package-lock.json +++ b/scripts/package-lock.json @@ -10,11 +10,12 @@ "license": "MIT", "dependencies": { "firebase-admin": "^13.0.2", - "inquirer": "^12.3.0" + "inquirer": "^12.3.0", + "p-queue": "^8.0.1" }, "engines": { - "node": ">=20", - "npm": ">=10" + "node": "20", + "npm": "10" } }, "node_modules/@fastify/busboy": { @@ -1012,6 +1013,12 @@ "node": ">=6" } }, + "node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", + "license": "MIT" + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -1689,6 +1696,34 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-queue": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.0.1.tgz", + "integrity": "sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.1", + "p-timeout": "^6.1.2" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.3.tgz", + "integrity": "sha512-UJUyfKbwvr/uZSV6btANfb+0t/mOhKV/KXcCUTp8FcQI+v/0d+wXqH4htrW0E4rR6WiEO/EPvUFiV9D5OI4vlw==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/proto3-json-serializer": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.2.tgz", @@ -2871,6 +2906,11 @@ "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", "optional": true }, + "eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==" + }, "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -3345,6 +3385,20 @@ "yocto-queue": "^0.1.0" } }, + "p-queue": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.0.1.tgz", + "integrity": "sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==", + "requires": { + "eventemitter3": "^5.0.1", + "p-timeout": "^6.1.2" + } + }, + "p-timeout": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.3.tgz", + "integrity": "sha512-UJUyfKbwvr/uZSV6btANfb+0t/mOhKV/KXcCUTp8FcQI+v/0d+wXqH4htrW0E4rR6WiEO/EPvUFiV9D5OI4vlw==" + }, "proto3-json-serializer": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.2.tgz", diff --git a/scripts/package.json b/scripts/package.json index f6af7be..8793765 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -8,7 +8,8 @@ "main": "src/index.js", "dependencies": { "firebase-admin": "^13.0.2", - "inquirer": "^12.3.0" + "inquirer": "^12.3.0", + "p-queue": "^8.0.1" }, "engines": { "node": "20",