docs: add supabase guide (#297)

* docs: add supabase guide * feat: add support for external connection poolers * chore: use .env symlink in database package * chore: remove database .env.example * docs: finish supabase pooling section * docs: fix small typo * docs: replace prisma with app wording * docs: fix up supabase pooling section * fix: deleted too much in contributing docs * docs: change pooling heading * docs: reduce linux shaming * docs: include direct connection url in fly.io
triggerdotdev · Aug 11, 2023 · 603ffbc · 603ffbc
1 parent beef776
commit 603ffbc
Show file tree

Hide file tree

Showing 17 changed files with 87 additions and 17 deletions.
diff --git a/.env.example b/.env.example
@@ -3,7 +3,10 @@ SESSION_SECRET=abcdef1234
 MAGIC_LINK_SECRET=abcdef1234
 ENCRYPTION_KEY=ae13021afef0819c3a307ad487071c06 # Must be a random 16 byte hex string. You can generate an encryption key by running `openssl rand -hex 16` in your terminal
 LOGIN_ORIGIN=http://localhost:3030
-DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres?schema=public 
+DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres?schema=public
+# This sets the URL used for direct connections to the database and should only be needed in limited circumstances
+# See: https://www.prisma.io/docs/reference/api-reference/prisma-schema-reference#fields:~:text=the%20shadow%20database.-,directUrl,-No
+DIRECT_URL=${DATABASE_URL}
 REMIX_APP_PORT=3030
 APP_ENV=development
 APP_ORIGIN=http://localhost:3030

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -122,7 +122,6 @@ jobs:
           # Setup environment variables
           cp ./.env.example ./.env
           cp ./examples/nextjs-test/.env.example ./examples/nextjs-test/.env.local
-          cp ./packages/database/.env.example ./packages/database/.env
 
           # Build packages
           pnpm run build --filter @examples/nextjs-test^...

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -37,11 +37,11 @@ branch are tagged into a release monthly.
    ```
    pnpm i
    ```
-4. Create your `.env` files
+4. Create your `.env` file
    ```
-   cp .env.example .env && cp packages/database/.env.example packages/database/.env
+   cp .env.example .env
    ```
-5. Open the root `.env` file and generate a new value for `ENCRYPTION_KEY`:
+5. Open it and generate a new value for `ENCRYPTION_KEY`:
 
    `ENCRYPTION_KEY` is used to two-way encrypt OAuth access tokens and so you'll probably want to actually generate a unique value, and it must be a random 16 byte hex string. You can generate one with the following command:
 
@@ -178,7 +178,6 @@ To run the end-to-end tests, follow the steps below:
 ```sh
 cp ./.env.example ./.env
 cp ./examples/nextjs-test/.env.example ./examples/nextjs-test/.env.local
-cp ./packages/database/.env.example ./packages/database/.env
 ```
 
 2. Set up dependencies

diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts
@@ -94,6 +94,7 @@ function getClient() {
     datasources: {
       db: {
         url: DATABASE_URL,
+        // We can't set directUrl here, and we don't have to
       },
     },
     log: [

diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts
@@ -4,6 +4,7 @@ import { SecretStoreOptionsSchema } from "./services/secrets/secretStore.server"
 const EnvironmentSchema = z.object({
   NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]),
   DATABASE_URL: z.string(),
+  DIRECT_URL: z.string(),
   SESSION_SECRET: z.string(),
   MAGIC_LINK_SECRET: z.string(),
   ENCRYPTION_KEY: z.string(),

diff --git a/apps/webapp/app/services/worker.server.ts b/apps/webapp/app/services/worker.server.ts
@@ -94,6 +94,7 @@ function getWorkerQueue() {
       connectionString: env.DATABASE_URL,
       concurrency: 5,
       pollInterval: 1000,
+      noPreparedStatements: env.DATABASE_URL !== env.DIRECT_URL,
     },
     schema: workerCatalog,
     recurringTasks: {

diff --git a/docker/dev-compose.yml b/docker/dev-compose.yml
@@ -32,6 +32,7 @@ services:
       - db
     environment:
       DATABASE_URL: postgres://postgres:postgres@db:5432/postgres?schema=public
+      DIRECT_URL: postgres://postgres:postgres@db:5432/postgres?schema=public
       SESSION_SECRET: secret123
       MAGIC_LINK_SECRET: secret123
       REMIX_APP_PORT: 3030

diff --git a/docs/documentation/guides/self-hosting/flyio.mdx b/docs/documentation/guides/self-hosting/flyio.mdx
@@ -19,6 +19,10 @@ You should fork the repository before starting so you can make changes and commi
 brew install flyctl
 ```
 
+```sh Linux
+curl -L https://fly.io/install.sh | sh
+```
+
 ```sh Windows
 pwsh -Command "iwr https://fly.io/install.ps1 -useb | iex"
 ```
@@ -41,6 +45,8 @@ fly launch
 
 2. Follow the prompts by `fly launch` and make sure to answer them in the following way:
 
+<Warning>Do NOT get rid of your terminal output, we will need your database connection string shortly.</Warning>
+
 ```sh
 ? Would you like to copy its configuration to the new app? Yes
 ? Choose an app name (leaving blank will default to 'trigger-v2-fly-demo') <enter your preferred app name here or leave blank>
@@ -54,18 +60,26 @@ fly launch
 
 ### Required
 
-`MAGIC_LINK_SECRET`, `SESSION_SECRET` and `ENCRYPTION_KEY`
+1. `MAGIC_LINK_SECRET`, `SESSION_SECRET` and `ENCRYPTION_KEY`
 
 All of these secrets should be 16-byte random strings, which you can easily generate (and copy into your pasteboard) with the following command:
 
 ```sh
 openssl rand -hex 16 | pbcopy
 ```
 
-`LOGIN_ORIGIN` and `APP_ORIGIN`
+2. `LOGIN_ORIGIN` and `APP_ORIGIN`
 
 Both of these secrets should be set to the base URL of your fly application. For example `https://trigger-v2-fly-demo.fly.dev`
 
+3. `DIRECT_URL`
+
+This needs to be set to the database connection string that was printed to your terminal after the creation step above:
+
+```sh
+Connection string: postgres://postgres:<PASSWORD>@<fly db name>.flycast:5432
+```
+
 ### Optional
 
 `AUTH_GITHUB_CLIENT_ID` and `AUTH_GITHUB_CLIENT_SECRET`
@@ -97,6 +111,7 @@ fly secrets set \
   SESSION_SECRET=<random string> \
   LOGIN_ORIGIN="https://<fly app name>.fly.dev" \
   APP_ORIGIN="https://<fly app name>.fly.dev" \
+  DIRECT_URL="postgres://postgres:<PASSWORD>@<fly db name>.flycast:5432" \
   FROM_EMAIL="Acme Inc. <hello@yourdomain.com>" \
   REPLY_TO_EMAIL="Acme Inc. <reply@yourdomain.com>" \
   RESEND_API_KEY=<your API Key> \

diff --git a/docs/documentation/guides/self-hosting/supabase.mdx b/docs/documentation/guides/self-hosting/supabase.mdx
@@ -53,7 +53,7 @@ ENCRYPTION_KEY=1189c93e399856a2a9a1454496171b2e
   <img src="/images/supabase-project-setup.png" alt="Supabase - Project Setup" />
 </Frame>
 
-5. Navigate to Project `Settings -> Database` and copy your database connection string
+5. Navigate to `Project Settings -> Database` and copy your database connection string
 
 <Frame>
   <img src="/images/supabase-db-url.png" alt="Supabase - Database URL" />
@@ -82,6 +82,7 @@ SESSION_SECRET=8d92078940c89588fc8b6f5481f2c6e0
 ENCRYPTION_KEY=1189c93e399856a2a9a1454496171b2e
 
 DATABASE_URL=postgresql://postgres:<PASSWORD>@db.<ID>.supabase.co:5432/postgres?schema=triggerdotdev
+DIRECT_URL=${DATABASE_URL}
 
 NODE_ENV=development
 RUNTIME_PLATFORM=docker-compose
@@ -136,15 +137,53 @@ docker compose up
 
 8. Congratulations, you're all set up and ready to go with Supabase and Docker! 🚀
 
-{/* TODO: add pooling setup */}
-
 ## Bonus: Connection pooling
 
+### What is connection pooling?
+
 <Frame caption="Connection Pool - © 2023 Supabase, Apache License 2.0">
   <img src="/images/supabase-connection-pool.png" alt="Supabase - Connection Pool" />
 </Frame>
 
-...coming soon!
+When running on a single server, our app can be trusted to manage its own connections - it knows how many are open and can process its internal queue as needed.
+
+<Tooltip tip="🐝">Serverless</Tooltip> changes things. Here, many concurrent app instances are started, each unaware of the others' connections. This can lead to a huge - potentially fatal - increase in connections straight to your DB.
+
+<Info>
+  PostgreSQL has a [default connection limit](https://www.postgresql.org/docs/current/runtime-config-connection.html#:~:text=at%20server%20start.-,max_connections,-(integer)) of 100.
+</Info>
+
+**External connection pools** solve this by putting an additional layer between client and server, which essentially works like a message queue that gets processed as limits allow. The clients are blind to this and can keep sending their requests as before.
+
+### Enable connection pooling
+
+1. Copy and paste the connection string from earlier to `DIRECT_URL`
+
+```sh .env (excerpt)
+...
+DIRECT_URL=postgresql://postgres:<PASSWORD>@db.<ID>.supabase.co:5432/postgres?schema=triggerdotdev
+...
+```
+
+2. Navigate to `Project Settings -> Database` and copy your connection pool URL
+
+<Frame>
+  <img src="/images/supabase-pool-url.png" alt="Supabase - Connection Pool URL" />
+</Frame>
+
+3. Paste it into your `.env` file next to `DATABASE_URL`
+
+4. Append `?schema=triggerdotdev&pgbouncer=true` and insert your password
+
+```sh .env (excerpt)
+...
+# notice the differing ports and additional query param
+DATABASE_URL=postgresql://postgres:<PASSWORD>@db.<ID>.supabase.co:6543/postgres?schema=triggerdotdev&pgbouncer=true
+DIRECT_URL=postgresql://postgres:<PASSWORD>@db.<ID>.supabase.co:5432/postgres?schema=triggerdotdev
+...
+```
+
+5. All done! You can now enjoy the benefits of connection pooling ⚡️
 
 ## Next steps
 

diff --git a/docs/documentation/guides/tunneling-platform.mdx b/docs/documentation/guides/tunneling-platform.mdx
@@ -18,6 +18,14 @@ There are a few ways to do this, but we recommend using [ngrok](https://ngrok.co
 brew install ngrok/ngrok/ngrok
 ```
 
+```sh Linux
+curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | \
+  sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null && \
+  echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | \
+  sudo tee /etc/apt/sources.list.d/ngrok.list && \
+  sudo apt update && sudo apt install ngrok
+```
+
 ```sh Windows
 choco install ngrok
 ```

diff --git a/docs/images/supabase-pool-url.png b/docs/images/supabase-pool-url.png
diff --git a/packages/database/.env b/packages/database/.env
@@ -0,0 +1 @@
+../../.env
diff --git a/packages/database/.env.example b/packages/database/.env.example
diff --git a/packages/database/.gitignore b/packages/database/.gitignore
@@ -1,3 +1,3 @@
 node_modules
-# Keep environment variables out of version control
-.env
+# Ensure the .env symlink is not removed by accident
+!.env
diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma
@@ -1,6 +1,7 @@
 datasource db {
-  provider = "postgresql"
-  url      = env("DATABASE_URL")
+  provider  = "postgresql"
+  url       = env("DATABASE_URL")
+  directUrl = env("DIRECT_URL")
 }
 
 generator client {

diff --git a/tests/utils.ts b/tests/utils.ts
@@ -9,6 +9,7 @@ export const setDB = async (cb: SetDBCallback) => {
     datasources: {
       db: {
         url: DATABASE_URL,
+        // We can't set directUrl here, and we don't have to
       },
     },
   });

diff --git a/turbo.json b/turbo.json
@@ -81,6 +81,7 @@
     "REMIX_APP_PORT",
     "CI",
     "DATABASE_URL",
+    "DIRECT_URL",
     "SESSION_SECRET",
     "APP_ORIGIN",
     "LOGIN_ORIGIN",