Add support for the built-in Trigger.dev tunnel for local development (…

…#795)

* Add support for the built-in Trigger.dev tunnel for local development

* Remove the test script from yalt

* Add to the README

* Improve the outdated packages warning

.changeset/config.json

@@ -14,6 +14,8 @@
   "ignore": [
     "webapp",
     "emails",
+    "proxy",
+    "yalt",
     "@trigger.dev/database"
   ],
   "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {

.changeset/new-panthers-do.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/yalt": patch
+"@trigger.dev/cli": patch
+---
+
+Add support for the built-in Trigger.dev tunnel (Yalt)

apps/webapp/app/env.server.ts

@@ -64,6 +64,9 @@ const EnvironmentSchema = z.object({
 
   DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(10),
   DEFAULT_DEV_ENV_EXECUTION_ATTEMPTS: z.coerce.number().int().positive().default(1),
+
+  TUNNEL_HOST: z.string().optional(),
+  TUNNEL_SECRET_KEY: z.string().optional(),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/routes/api.v1.tunnels.ts

@@ -0,0 +1,69 @@
+import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { env } from "~/env.server";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { YaltApiClient } from "@trigger.dev/yalt";
+import { logger } from "~/services/logger.server";
+import { prisma } from "~/db.server";
+
+// This is for HEAD requests to check if the API supports tunneling
+export async function loader({ request }: LoaderFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  if (!env.TUNNEL_HOST || !env.TUNNEL_SECRET_KEY) {
+    return json({ error: "Tunneling is not supported" }, { status: 501 });
+  }
+
+  return json({ ok: true });
+}
+
+export async function action({ request }: LoaderFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  if (authenticationResult.environment.type !== "DEVELOPMENT") {
+    return json({ error: "Tunneling is only supported in development" }, { status: 501 });
+  }
+
+  if (!env.TUNNEL_HOST || !env.TUNNEL_SECRET_KEY) {
+    return json({ error: "Tunneling is not supported" }, { status: 501 });
+  }
+
+  const yaltClient = new YaltApiClient(env.TUNNEL_HOST, env.TUNNEL_SECRET_KEY);
+
+  let tunnelId = authenticationResult.environment.tunnelId;
+
+  if (!tunnelId) {
+    try {
+      tunnelId = await yaltClient.createTunnel();
+
+      await prisma.runtimeEnvironment.update({
+        where: {
+          id: authenticationResult.environment.id,
+        },
+        data: {
+          tunnelId,
+        },
+      });
+    } catch (error) {
+      logger.error("Failed to create tunnel", { error });
+
+      return json({ error: "Failed to create tunnel" }, { status: 500 });
+    }
+  }
+
+  if (!tunnelId) {
+    return json({ error: "Failed to create tunnel" }, { status: 500 });
+  }
+
+  return json({ url: yaltClient.connectUrl(tunnelId) });
+}

apps/webapp/package.json

@@ -72,6 +72,7 @@
     "@trigger.dev/core-backend": "workspace:*",
     "@trigger.dev/database": "workspace:*",
     "@trigger.dev/sdk": "workspace:*",
+    "@trigger.dev/yalt": "workspace:*",
     "@types/pg": "8.6.6",
     "@uiw/react-codemirror": "^4.19.5",
     "@whatwg-node/fetch": "^0.9.14",

apps/webapp/remix.config.js

@@ -15,6 +15,7 @@ module.exports = {
     "@trigger.dev/core-backend",
     "@trigger.dev/sdk",
     "@trigger.dev/billing",
+    "@trigger.dev/yalt",
     "emails",
     "highlight.run",
     "random-words",
@@ -24,6 +25,7 @@ module.exports = {
       "../../packages/core/src/**/*",
       "../../packages/core-backend/src/**/*",
       "../../packages/trigger-sdk/src/**/*",
+      "../../packages/yalt/src/**/*",
       "../../packages/emails/src/**/*",
     ];
   },

apps/webapp/tsconfig.json

@@ -27,6 +27,8 @@
       "@trigger.dev/core-backend/*": ["../../packages/core-backend/src/*"],
       "@trigger.dev/database": ["../../packages/database/src/index"],
       "@trigger.dev/database/*": ["../../packages/database/src/*"],
+      "@trigger.dev/yalt": ["../../packages/yalt/src/index"],
+      "@trigger.dev/yalt/*": ["../../packages/yalt/src/*"],
       "emails": ["../../packages/emails/src/index"],
       "emails/*": ["../../packages/emails/src/*"]
     },

apps/yalt/.dev.vars.example

@@ -0,0 +1 @@
+SECRET_KEY=

apps/yalt/.editorconfig

@@ -0,0 +1,13 @@
+# http://editorconfig.org
+root = true
+
+[*]
+indent_style = tab
+tab_width = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.yml]
+indent_style = space

apps/yalt/.gitignore

@@ -0,0 +1,172 @@
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+\*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+\*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+\*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+\*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.cache
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+.cache/
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+.cache
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.\*
+
+# wrangler project
+
+.dev.vars
+.wrangler/

apps/yalt/.prettierrc

@@ -0,0 +1,6 @@
+{
+	"printWidth": 140,
+	"singleQuote": true,
+	"semi": true,
+	"useTabs": true
+}

apps/yalt/README.md

@@ -0,0 +1,26 @@
+## Trigger.dev Yalt Server
+
+Yalt (Yet Another Local Tunnel) is the Trigger.dev tunneling service that powers the local development of Trigger.dev cloud users.
+
+## Why?
+
+The Trigger.dev server communicates with user endpoints over HTTP, so during local development we need a way for the Trigger.dev server to make HTTP requests over the public internet to the user's local machine. This is accomplished via a tunneling service like ngrok, which we've been using up until now. Unfortunately, the ngrok free plan has pretty aggressive rate limits and some Trigger.dev users have faced issues with their local jobs not running/working because of this. Yalt.dev is our solution.
+
+## How does it work?
+
+Yalt.dev is a Cloudflare Worker that uses Durable Objects to persist a websocket connection from the `@trigger.dev/cli dev` command and proxies requests through the websocket connection and then returns responses from the websocket connection.
+
+- There is an admin API available at `admin.trigger.dev` that allows tunnels to be created (authenticated via a SECRET_KEY)
+- The Cloudflare Worker has a wildcard subdomain route `*.yalt.dev/*`
+- When the client receives the tunnel ID, they can connect to the server via `wss://${tunnelId}.yalt.dev/connect`
+- Now, requests to `https://${tunnelId}.yalt.dev/api/trigger` are sent to the Durable Object (`YaltConnection`) using the subdomain/tunnelId
+- Requests are serialized to a JSON string and sent to the WebSocket
+- The WebSocket client running in `@trigger.dev/cli dev` receives the request message and makes a real request to the local dev server
+- The `@trigger.dev/cli dev` serializes the response and sends it back to the server.
+- The server responds to the original request
+
+Along with this server there is a package called `@trigger.dev/yalt` that has shared code and is used in:
+
+- Yalt.dev server (this project)
+- `@trigger.dev/cli dev` command
+- The Trigger.dev server (to create the tunnels)

apps/yalt/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "yalt",
+  "version": "0.0.1",
+  "private": true,
+  "scripts": {
+    "deploy": "wrangler deploy",
+    "dev": "wrangler dev --port 8787",
+    "start": "wrangler dev"
+  },
+  "devDependencies": {
+    "@cloudflare/workers-types": "~4.20231121.0",
+    "wrangler": "^3.20.0"
+  },
+  "dependencies": {
+    "@trigger.dev/yalt": "workspace:*"
+  }
+}