Archive games after 24h in GCS and restore automatically (#172)

* Archive games after 24h in GCS and restore automatically

This reduces storage costs from $5/GB/month to about $0.02/GB/month (250x) with almost no downside. It also uses gzip which provides another 3x compression ratio, reducing cost again. So what was costing $150/month previously to store 10 million games, and required us to drop the database, now would only cost $0.20/month.

Need to test thoroughly in setwihfriends-dev before I can promote to production. Don't want to lose a bunch of data!

* Fix lint warning

functions/src/index.ts

@@ -1,10 +1,12 @@
 import { initializeApp } from "firebase-admin/app";
 import { getAuth } from "firebase-admin/auth";
 import { type DataSnapshot, getDatabase } from "firebase-admin/database";
+import { getStorage } from "firebase-admin/storage";
 import * as functions from "firebase-functions/v1";
 import Stripe from "stripe";
 
 import { GameMode, findSet, generateDeck, replayEvents } from "./game";
+import { databaseIterator, gzip } from "./utils";
 
 initializeApp(); // Sets the default Firebase app.
 
@@ -228,7 +230,7 @@ export const createGame = functions.https.onCall(async (data, context) => {
 
   const userId = context.auth.uid;
 
-  const oneHourAgo = Date.now() - 3600000;
+  const oneHourAgo = Date.now() - 3600 * 1000;
   const recentGameIds = await getDatabase()
     .ref(`userGames/${userId}`)
     .orderByValue()
@@ -416,3 +418,88 @@ export const handleStripe = functions.https.onRequest(async (req, res) => {
 
   res.status(200).end();
 });
+
+/**
+ * Archive a game state from RTDB to GCS, reducing the storage tier.
+ * Returns whether the state was found in the database.
+ */
+async function archiveGameState(gameId: string): Promise<boolean> {
+  const snap = await getDatabase().ref(`gameData/${gameId}`).get();
+  if (!snap.exists()) {
+    return false; // Game state is not present in the database, maybe racy?
+  }
+
+  const jsonBlob = JSON.stringify(snap.val());
+  const gzippedBlob = await gzip.compress(jsonBlob);
+
+  await getStorage()
+    .bucket()
+    .file(`gameData/${gameId}.json.gz`)
+    .save(gzippedBlob);
+
+  // After archiving, we remove the game state from the database.
+  await getDatabase().ref(`gameData/${gameId}`).remove();
+  return true;
+}
+
+/** Restore a game state in GCS to RTDB so it can be read from the client. */
+async function restoreGameState(gameId: string): Promise<boolean> {
+  const file = getStorage().bucket().file(`gameData/${gameId}.json.gz`);
+  let gzippedBlob: Buffer;
+  try {
+    [gzippedBlob] = await file.download();
+  } catch (error: any) {
+    // File was not present.
+    if (error.code === 404) return false;
+    throw error;
+  }
+  const jsonBlob = await gzip.decompress(gzippedBlob);
+
+  const gameData = JSON.parse(jsonBlob.toString());
+  await getDatabase().ref(`gameData/${gameId}`).set(gameData);
+  return true;
+}
+
+/** Try to fetch a game state that's not present, restoring if needed. */
+export const fetchStaleGame = functions.https.onCall(async (data, context) => {
+  if (!context.auth) {
+    throw new functions.https.HttpsError(
+      "failed-precondition",
+      "The function must be called while authenticated.",
+    );
+  }
+
+  const gameId = data.gameId;
+  if (
+    !(typeof gameId === "string") ||
+    gameId.length === 0 ||
+    gameId.length > MAX_GAME_ID_LENGTH
+  ) {
+    throw new functions.https.HttpsError(
+      "invalid-argument",
+      "The function must be called with argument `gameId` to be fetched at `/games/:gameId`.",
+    );
+  }
+
+  try {
+    const restored = await restoreGameState(gameId);
+    return { restored };
+  } catch (error: any) {
+    throw new functions.https.HttpsError("internal", error.message);
+  }
+});
+
+/** Archive stale game states to GCS for cost savings. */
+export const archiveStaleGames = functions.pubsub
+  .schedule("every 1 hours")
+  .onRun(async (context) => {
+    const cutoff = Date.now() - 86400 * 1000; // 24 hours ago
+
+    for await (const [gameId] of databaseIterator("gameData")) {
+      const game = await getDatabase().ref(`games/${gameId}`).get();
+      if (game.val().createdAt < cutoff) {
+        console.log(`Archiving stale game state for ${gameId}`);
+        await archiveGameState(gameId);
+      }
+    }
+  });

functions/src/utils.ts

@@ -0,0 +1,59 @@
+import { type DataSnapshot, getDatabase } from "firebase-admin/database";
+import * as zlib from "node:zlib";
+
+/** Promises API wrapper around the 'zlib' Node library. */
+export const gzip = {
+  compress(
+    input: string | Buffer | ArrayBuffer | Uint8Array | DataView,
+  ): Promise<Buffer> {
+    return new Promise((resolve, reject) => {
+      zlib.gzip(input, (err, result) => {
+        if (err) reject(err);
+        else resolve(result);
+      });
+    });
+  },
+  decompress(input: Buffer): Promise<Buffer> {
+    return new Promise((resolve, reject) => {
+      zlib.gunzip(input, (err, result) => {
+        if (err) reject(err);
+        else resolve(result);
+      });
+    });
+  },
+};
+
+/**
+ * Iterate through a reference in the database, returning the children ordered
+ * by key in batches.
+ */
+export async function* databaseIterator(
+  path: string,
+  batchSize = 1000,
+): AsyncGenerator<[string, DataSnapshot]> {
+  let lastKey = null;
+  while (true) {
+    const snap = lastKey
+      ? await getDatabase()
+          .ref(path)
+          .orderByKey()
+          .startAfter(lastKey)
+          .limitToFirst(batchSize)
+          .get()
+      : await getDatabase()
+          .ref(path)
+          .orderByKey()
+          .limitToFirst(batchSize)
+          .get();
+    if (!snap.exists()) return;
+
+    const childKeys: string[] = [];
+    snap.forEach((child) => {
+      childKeys.push(child.key);
+      lastKey = child.key;
+    });
+    for (const key of childKeys) {
+      yield [key, snap.child(key)];
+    }
+  }
+}

rundev.js

@@ -64,19 +64,31 @@ const pubsub = new PubSub({
   projectId: "setwithfriends-dev",
 });
 
-const pubsubInterval = setInterval(async () => {
-  await pubsub
-    .topic("firebase-schedule-clearConnections")
-    .publishMessage({ json: {} });
-}, 60 * 1000); // every minute
+// This is a workaround for the Pub/Sub emulator not supporting scheduled functions.
+// https://github.com/firebase/firebase-tools/issues/2034
+const pubsubIntervals = [
+  setInterval(async () => {
+    await pubsub
+      .topic("firebase-schedule-clearConnections")
+      .publishMessage({ json: {} });
+  }, 60 * 1000), // every minute
+
+  setInterval(async () => {
+    await pubsub
+      .topic("firebase-schedule-archiveStaleGames")
+      .publishMessage({ json: {} });
+  }, 3600 * 1000), // every hour
+];
 
 let shutdownCalled = false;
 
 async function shutdown() {
   if (shutdownCalled) return;
   shutdownCalled = true;
 
-  clearInterval(pubsubInterval);
+  for (const interval of pubsubIntervals) {
+    clearInterval(interval);
+  }
 
   const waitForChild = (p) => new Promise((resolve) => p.on("exit", resolve));
   await Promise.all([

scripts/src/calcStats.js

@@ -1,3 +1,4 @@
+// Note: This is deprecated.
 import { getDatabase } from "firebase-admin/database";
 
 const batchSize = 1000;

scripts/src/fixGames.js

@@ -1,3 +1,4 @@
+// Note: This is deprecated.
 import assert from "assert";
 import { getDatabase } from "firebase-admin/database";
 

src/firebase.js

@@ -26,5 +26,6 @@ const functions = firebase.functions();
 export const createGame = functions.httpsCallable("createGame");
 export const customerPortal = functions.httpsCallable("customerPortal");
 export const finishGame = functions.httpsCallable("finishGame");
+export const fetchStaleGame = functions.httpsCallable("fetchStaleGame");
 
 export default firebase;

src/pages/GamePage.js

@@ -20,7 +20,7 @@ import Loading from "../components/Loading";
 import SnackContent from "../components/SnackContent";
 import User from "../components/User";
 import { SettingsContext, UserContext } from "../context";
-import firebase, { createGame, finishGame } from "../firebase";
+import firebase, { createGame, fetchStaleGame, finishGame } from "../firebase";
 import useFirebaseRef from "../hooks/useFirebaseRef";
 import {
   checkSet,
@@ -81,6 +81,7 @@ function GamePage() {
   const [selected, setSelected] = useState([]);
   const [snack, setSnack] = useState({ open: false });
   const [numHints, setNumHints] = useState(0);
+  const [fetchingStaleGame, setFetchingStaleGame] = useState("not-stale");
 
   const [game, loadingGame] = useFirebaseRef(`games/${gameId}`);
   const [gameData, loadingGameData] = useFirebaseRef(`gameData/${gameId}`);
@@ -128,9 +129,35 @@ function GamePage() {
     }
   });
 
+  // Try to fetch the game from cloud storage, if archived due to being stale.
+  useEffect(() => {
+    if (!loadingGame && !loadingGameData) {
+      if (game && gameData) {
+        if (fetchingStaleGame !== "not-stale") {
+          setFetchingStaleGame("not-stale");
+        }
+      } else {
+        if (fetchingStaleGame === "not-stale") {
+          setFetchingStaleGame("fetching");
+          (async () => {
+            // On success, the database should automatically update with game state.
+            const { restored } = await fetchStaleGame({ gameId });
+            if (!restored) {
+              setFetchingStaleGame("failed");
+            }
+          })();
+        }
+      }
+    }
+  }, [loadingGame, loadingGameData, game, gameData, fetchingStaleGame, gameId]);
+
   if (redirect) return <Navigate push to={redirect} />;
 
-  if (loadingGame || loadingGameData) {
+  if (
+    loadingGame ||
+    loadingGameData ||
+    ((!game || !gameData) && fetchingStaleGame !== "failed")
+  ) {
     return <LoadingPage />;
   }