From ee37b0deb84e2a54c7287f64cfbb332c1ca847b4 Mon Sep 17 00:00:00 2001 From: Eric Zhang Date: Sun, 22 Dec 2024 20:33:13 -0600 Subject: [PATCH 1/2] Archive games after 24h in GCS and restore automatically This reduces storage costs from $5/GB/month to about $0.02/GB/month (250x) with almost no downside. It also uses gzip which provides another 3x compression ratio, reducing cost again. So what was costing $150/month previously to store 10 million games, and required us to drop the database, now would only cost $0.20/month. Need to test thoroughly in setwihfriends-dev before I can promote to production. Don't want to lose a bunch of data! --- functions/src/index.ts | 89 +++++++++++++++++++++++++++++++++++++++- functions/src/utils.ts | 59 ++++++++++++++++++++++++++ rundev.js | 24 ++++++++--- scripts/src/calcStats.js | 1 + scripts/src/fixGames.js | 1 + src/firebase.js | 1 + src/pages/GamePage.js | 31 +++++++++++++- 7 files changed, 197 insertions(+), 9 deletions(-) create mode 100644 functions/src/utils.ts diff --git a/functions/src/index.ts b/functions/src/index.ts index 0230425..d87d1c9 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -1,10 +1,12 @@ import { initializeApp } from "firebase-admin/app"; import { getAuth } from "firebase-admin/auth"; import { type DataSnapshot, getDatabase } from "firebase-admin/database"; +import { getStorage } from "firebase-admin/storage"; import * as functions from "firebase-functions/v1"; import Stripe from "stripe"; import { GameMode, findSet, generateDeck, replayEvents } from "./game"; +import { databaseIterator, gzip } from "./utils"; initializeApp(); // Sets the default Firebase app. @@ -228,7 +230,7 @@ export const createGame = functions.https.onCall(async (data, context) => { const userId = context.auth.uid; - const oneHourAgo = Date.now() - 3600000; + const oneHourAgo = Date.now() - 3600 * 1000; const recentGameIds = await getDatabase() .ref(`userGames/${userId}`) .orderByValue() @@ -416,3 +418,88 @@ export const handleStripe = functions.https.onRequest(async (req, res) => { res.status(200).end(); }); + +/** + * Archive a game state from RTDB to GCS, reducing the storage tier. + * Returns whether the state was found in the database. + */ +async function archiveGameState(gameId: string): Promise { + const snap = await getDatabase().ref(`gameData/${gameId}`).get(); + if (!snap.exists()) { + return false; // Game state is not present in the database, maybe racy? + } + + const jsonBlob = JSON.stringify(snap.val()); + const gzippedBlob = await gzip.compress(jsonBlob); + + await getStorage() + .bucket() + .file(`gameData/${gameId}.json.gz`) + .save(gzippedBlob); + + // After archiving, we remove the game state from the database. + await getDatabase().ref(`gameData/${gameId}`).remove(); + return true; +} + +/** Restore a game state in GCS to RTDB so it can be read from the client. */ +async function restoreGameState(gameId: string): Promise { + const file = getStorage().bucket().file(`gameData/${gameId}.json.gz`); + let gzippedBlob: Buffer; + try { + [gzippedBlob] = await file.download(); + } catch (error: any) { + // File was not present. + if (error.code === 404) return false; + throw error; + } + const jsonBlob = await gzip.decompress(gzippedBlob); + + const gameData = JSON.parse(jsonBlob.toString()); + await getDatabase().ref(`gameData/${gameId}`).set(gameData); + return true; +} + +/** Try to fetch a game state that's not present, restoring if needed. */ +export const fetchStaleGame = functions.https.onCall(async (data, context) => { + if (!context.auth) { + throw new functions.https.HttpsError( + "failed-precondition", + "The function must be called while authenticated.", + ); + } + + const gameId = data.gameId; + if ( + !(typeof gameId === "string") || + gameId.length === 0 || + gameId.length > MAX_GAME_ID_LENGTH + ) { + throw new functions.https.HttpsError( + "invalid-argument", + "The function must be called with argument `gameId` to be fetched at `/games/:gameId`.", + ); + } + + try { + const restored = await restoreGameState(gameId); + return { restored }; + } catch (error: any) { + throw new functions.https.HttpsError("internal", error.message); + } +}); + +/** Archive stale game states to GCS for cost savings. */ +export const archiveStaleGames = functions.pubsub + .schedule("every 1 hours") + .onRun(async (context) => { + const cutoff = Date.now() - 86400 * 1000; // 24 hours ago + + for await (const [gameId] of databaseIterator("gameData")) { + const game = await getDatabase().ref(`games/${gameId}`).get(); + if (game.val().createdAt < cutoff) { + console.log(`Archiving stale game state for ${gameId}`); + await archiveGameState(gameId); + } + } + }); diff --git a/functions/src/utils.ts b/functions/src/utils.ts new file mode 100644 index 0000000..1729e6a --- /dev/null +++ b/functions/src/utils.ts @@ -0,0 +1,59 @@ +import { type DataSnapshot, getDatabase } from "firebase-admin/database"; +import * as zlib from "node:zlib"; + +/** Promises API wrapper around the 'zlib' Node library. */ +export const gzip = { + compress( + input: string | Buffer | ArrayBuffer | Uint8Array | DataView, + ): Promise { + return new Promise((resolve, reject) => { + zlib.gzip(input, (err, result) => { + if (err) reject(err); + else resolve(result); + }); + }); + }, + decompress(input: Buffer): Promise { + return new Promise((resolve, reject) => { + zlib.gunzip(input, (err, result) => { + if (err) reject(err); + else resolve(result); + }); + }); + }, +}; + +/** + * Iterate through a reference in the database, returning the children ordered + * by key in batches. + */ +export async function* databaseIterator( + path: string, + batchSize = 1000, +): AsyncGenerator<[string, DataSnapshot]> { + let lastKey = null; + while (true) { + const snap = lastKey + ? await getDatabase() + .ref(path) + .orderByKey() + .startAfter(lastKey) + .limitToFirst(batchSize) + .get() + : await getDatabase() + .ref(path) + .orderByKey() + .limitToFirst(batchSize) + .get(); + if (!snap.exists()) return; + + const childKeys: string[] = []; + snap.forEach((child) => { + childKeys.push(child.key); + lastKey = child.key; + }); + for (const key of childKeys) { + yield [key, snap.child(key)]; + } + } +} diff --git a/rundev.js b/rundev.js index a225c45..499403d 100644 --- a/rundev.js +++ b/rundev.js @@ -64,11 +64,21 @@ const pubsub = new PubSub({ projectId: "setwithfriends-dev", }); -const pubsubInterval = setInterval(async () => { - await pubsub - .topic("firebase-schedule-clearConnections") - .publishMessage({ json: {} }); -}, 60 * 1000); // every minute +// This is a workaround for the Pub/Sub emulator not supporting scheduled functions. +// https://github.com/firebase/firebase-tools/issues/2034 +const pubsubIntervals = [ + setInterval(async () => { + await pubsub + .topic("firebase-schedule-clearConnections") + .publishMessage({ json: {} }); + }, 60 * 1000), // every minute + + setInterval(async () => { + await pubsub + .topic("firebase-schedule-archiveStaleGames") + .publishMessage({ json: {} }); + }, 3600 * 1000), // every hour +]; let shutdownCalled = false; @@ -76,7 +86,9 @@ async function shutdown() { if (shutdownCalled) return; shutdownCalled = true; - clearInterval(pubsubInterval); + for (const interval of pubsubIntervals) { + clearInterval(interval); + } const waitForChild = (p) => new Promise((resolve) => p.on("exit", resolve)); await Promise.all([ diff --git a/scripts/src/calcStats.js b/scripts/src/calcStats.js index 29340c3..0258494 100644 --- a/scripts/src/calcStats.js +++ b/scripts/src/calcStats.js @@ -1,3 +1,4 @@ +// Note: This is deprecated. import { getDatabase } from "firebase-admin/database"; const batchSize = 1000; diff --git a/scripts/src/fixGames.js b/scripts/src/fixGames.js index 54826d6..ef07127 100644 --- a/scripts/src/fixGames.js +++ b/scripts/src/fixGames.js @@ -1,3 +1,4 @@ +// Note: This is deprecated. import assert from "assert"; import { getDatabase } from "firebase-admin/database"; diff --git a/src/firebase.js b/src/firebase.js index a75190d..68f07bb 100644 --- a/src/firebase.js +++ b/src/firebase.js @@ -26,5 +26,6 @@ const functions = firebase.functions(); export const createGame = functions.httpsCallable("createGame"); export const customerPortal = functions.httpsCallable("customerPortal"); export const finishGame = functions.httpsCallable("finishGame"); +export const fetchStaleGame = functions.httpsCallable("fetchStaleGame"); export default firebase; diff --git a/src/pages/GamePage.js b/src/pages/GamePage.js index 87a47d1..7e4fad7 100644 --- a/src/pages/GamePage.js +++ b/src/pages/GamePage.js @@ -20,7 +20,7 @@ import Loading from "../components/Loading"; import SnackContent from "../components/SnackContent"; import User from "../components/User"; import { SettingsContext, UserContext } from "../context"; -import firebase, { createGame, finishGame } from "../firebase"; +import firebase, { createGame, fetchStaleGame, finishGame } from "../firebase"; import useFirebaseRef from "../hooks/useFirebaseRef"; import { checkSet, @@ -81,6 +81,7 @@ function GamePage() { const [selected, setSelected] = useState([]); const [snack, setSnack] = useState({ open: false }); const [numHints, setNumHints] = useState(0); + const [fetchingStaleGame, setFetchingStaleGame] = useState("not-stale"); const [game, loadingGame] = useFirebaseRef(`games/${gameId}`); const [gameData, loadingGameData] = useFirebaseRef(`gameData/${gameId}`); @@ -128,9 +129,35 @@ function GamePage() { } }); + // Try to fetch the game from cloud storage, if archived due to being stale. + useEffect(() => { + if (!loadingGame && !loadingGameData) { + if (game && gameData) { + if (fetchingStaleGame !== "not-stale") { + setFetchingStaleGame("not-stale"); + } + } else { + if (fetchingStaleGame === "not-stale") { + setFetchingStaleGame("fetching"); + (async () => { + // On success, the database should automatically update with game state. + const { restored } = await fetchStaleGame({ gameId }); + if (!restored) { + setFetchingStaleGame("failed"); + } + })(); + } + } + } + }); + if (redirect) return ; - if (loadingGame || loadingGameData) { + if ( + loadingGame || + loadingGameData || + ((!game || !gameData) && fetchingStaleGame !== "failed") + ) { return ; } From 73e5c21c186c07978b574aa89e228e2574d8057d Mon Sep 17 00:00:00 2001 From: Eric Zhang Date: Sun, 22 Dec 2024 20:39:33 -0600 Subject: [PATCH 2/2] Fix lint warning --- src/pages/GamePage.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/GamePage.js b/src/pages/GamePage.js index 7e4fad7..e3364d2 100644 --- a/src/pages/GamePage.js +++ b/src/pages/GamePage.js @@ -149,7 +149,7 @@ function GamePage() { } } } - }); + }, [loadingGame, loadingGameData, game, gameData, fetchingStaleGame, gameId]); if (redirect) return ;