Introduce nativelink-bridge

This is a fast prototype for subscribing to the redis/dragonflydb "build_events" channel and decode them properly via protobuf and fires them via websocket to the browser.
TraceMachina · Nov 4, 2024 · c2363ab · c2363ab
1 parent ac44984
commit c2363ab
Show file tree

Hide file tree

Showing 19 changed files with 904 additions and 0 deletions.
diff --git a/flake.nix b/flake.nix
@@ -279,6 +279,8 @@
             };
           };
 
+        nativelink-bridge = pkgs.callPackage ./web/bridge/image.nix {inherit buildImage pullImage pkgs;};
+
         nativelink-worker-init = pkgs.callPackage ./tools/nativelink-worker-init.nix {inherit buildImage self nativelink-image;};
 
         rbe-autogen = pkgs.callPackage ./local-remote-execution/rbe-autogen.nix {
@@ -419,6 +421,7 @@
               nativelink-worker-init
               nativelink-x86_64-linux
               publish-ghcr
+              nativelink-bridge
               ;
             default = nativelink;
 

diff --git a/nativelink-config/examples/basic_bes.json b/nativelink-config/examples/basic_bes.json
@@ -0,0 +1,177 @@
+{
+  "stores": {
+    "AC_MAIN_STORE": {
+      "filesystem": {
+        "content_path": "/tmp/nativelink/data-worker-test/content_path-ac",
+        "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac",
+        "eviction_policy": {
+          "max_bytes": 100000000000
+        }
+      }
+    },
+    "BEP_STORE": {
+      "redis_store": {
+        "addresses": [
+          "redis://@localhost:6379/0"
+        ],
+        "response_timeout_s": 5,
+        "connection_timeout_s": 5,
+        "experimental_pub_sub_channel": "build_event",
+        "key_prefix": "nativelink:",
+        "mode": "standard"
+      }
+    },
+    "WORKER_FAST_SLOW_STORE": {
+      "fast_slow": {
+        "fast": {
+          "filesystem": {
+            "content_path": "/tmp/nativelink/data-worker-test/content_path-cas",
+            "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas",
+            "eviction_policy": {
+              "max_bytes": 100000000000
+            }
+          }
+        },
+        "slow": {
+          "noop": {}
+        }
+      }
+    }
+  },
+  "schedulers": {
+    "MAIN_SCHEDULER": {
+      "simple": {
+        "supported_platform_properties": {
+          "cpu_count": "minimum",
+          "memory_kb": "minimum",
+          "network_kbps": "minimum",
+          "disk_read_iops": "minimum",
+          "disk_read_bps": "minimum",
+          "disk_write_iops": "minimum",
+          "disk_write_bps": "minimum",
+          "shm_size": "minimum",
+          "gpu_count": "minimum",
+          "gpu_model": "exact",
+          "cpu_vendor": "exact",
+          "cpu_arch": "exact",
+          "cpu_model": "exact",
+          "kernel_version": "exact",
+          "OSFamily": "priority",
+          "container-image": "priority"
+        }
+      }
+    }
+  },
+  "workers": [
+    {
+      "local": {
+        "worker_api_endpoint": {
+          "uri": "grpc://127.0.0.1:50062"
+        },
+        "cas_fast_slow_store": "WORKER_FAST_SLOW_STORE",
+        "upload_action_result": {
+          "ac_store": "AC_MAIN_STORE"
+        },
+        "work_directory": "/tmp/nativelink/work",
+        "platform_properties": {
+          "cpu_count": {
+            "values": [
+              "16"
+            ]
+          },
+          "memory_kb": {
+            "values": [
+              "500000"
+            ]
+          },
+          "network_kbps": {
+            "values": [
+              "100000"
+            ]
+          },
+          "cpu_arch": {
+            "values": [
+              "x86_64"
+            ]
+          },
+          "OSFamily": {
+            "values": [
+              ""
+            ]
+          },
+          "container-image": {
+            "values": [
+              ""
+            ]
+          }
+        }
+      }
+    }
+  ],
+  "servers": [
+    {
+      "name": "public",
+      "listener": {
+        "http": {
+          "socket_address": "0.0.0.0:50052"
+        }
+      },
+      "services": {
+        "cas": {
+          "main": {
+            "cas_store": "WORKER_FAST_SLOW_STORE"
+          }
+        },
+        "ac": {
+          "main": {
+            "ac_store": "AC_MAIN_STORE"
+          }
+        },
+        "execution": {
+          "main": {
+            "cas_store": "WORKER_FAST_SLOW_STORE",
+            "scheduler": "MAIN_SCHEDULER"
+          }
+        },
+        "capabilities": {
+          "main": {
+            "remote_execution": {
+              "scheduler": "MAIN_SCHEDULER"
+            }
+          }
+        },
+        "bytestream": {
+          "cas_stores": {
+            "main": "WORKER_FAST_SLOW_STORE"
+          }
+        }
+      }
+    },
+    {
+      "name": "private_workers_servers",
+      "listener": {
+        "http": {
+          "socket_address": "0.0.0.0:50062"
+        }
+      },
+      "services": {
+        "experimental_prometheus": {
+          "path": "/metrics"
+        },
+        "experimental_bep": {
+          "store": "BEP_STORE"
+        },
+        "worker_api": {
+          "scheduler": "MAIN_SCHEDULER"
+        },
+        "admin": {},
+        "health": {
+          "path": "/status"
+        }
+      }
+    }
+  ],
+  "global": {
+    "max_open_files": 512
+  }
+}
diff --git a/tools/pre-commit-hooks.nix b/tools/pre-commit-hooks.nix
@@ -65,6 +65,7 @@ in {
 
       # Bun binary lockfile
       "web/platform/bun.lockb"
+      "web/bridge/bun.lockb"
     ];
     enable = true;
     types = ["binary"];

diff --git a/web/bridge/.env.example b/web/bridge/.env.example
@@ -0,0 +1,3 @@
+REDIS_URL=redis://localhost:6379
+NATIVELINK_PUB_SUB_CHANNEL=build_event
+POSTGRES_URL=
diff --git a/web/bridge/.gitignore b/web/bridge/.gitignore
@@ -0,0 +1,139 @@
+# Logs
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Caches
+.cache
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+pids
+_.pid
+_.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# IntelliJ based IDEs
+.idea
+
+# Finder (MacOS) folder config
+.DS_Store