[Breaking] Make NativeLink configurable via K8s-compatible yaml

TraceMachina · Nov 13, 2024 · 8f34db6 · 8f34db6
1 parent b6cf659
commit 8f34db6
Show file tree

Hide file tree

Showing 72 changed files with 4,160 additions and 829 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -35,10 +35,14 @@ enable_tokio_console = [
 nix = [
   "nativelink-worker/nix"
 ]
+crd = [
+  "nativelink-config/crd"
+]
 
 [dependencies]
 nativelink-error = { path = "nativelink-error" }
 nativelink-config = { path = "nativelink-config" }
+nativelink-controller = { path = "nativelink-controller" }
 nativelink-scheduler = { path = "nativelink-scheduler" }
 nativelink-service = { path = "nativelink-service" }
 nativelink-store = { path = "nativelink-store" }

diff --git a/deployment-examples/docker-compose/local-storage-cas.json b/deployment-examples/docker-compose/local-storage-cas.json
@@ -4,31 +4,31 @@
 // so objects are compressed, deduplicated and uses some in-memory
 // optimizations for certain hot paths.
 {
-  "stores": {
-    "CAS_MAIN_STORE": {
+  "stores": [
+    {
+      "name": "CAS_MAIN_STORE",
       "compression": {
         "compression_algorithm": {
           "lz4": {}
         },
         "backend": {
+          "name": "fs-cas",
           "filesystem": {
             "content_path": "~/.cache/nativelink/content_path-cas",
             "temp_path": "~/.cache/nativelink/tmp_path-cas",
             "eviction_policy": {
-              // 10gb.
-              "max_bytes": 10000000000,
+              "max_bytes": "10Gb",
             }
           }
         }
       }
-    },
-    "AC_MAIN_STORE": {
+    }, {
+      "name": "AC_MAIN_STORE",
       "filesystem": {
         "content_path": "~/.cache/nativelink/content_path-ac",
         "temp_path": "~/.cache/nativelink/tmp_path-ac",
         "eviction_policy": {
-          // 500mb.
-          "max_bytes": 500000000,
+          "max_bytes": "500Mb"
         }
       }
     }

diff --git a/deployment-examples/docker-compose/scheduler.json b/deployment-examples/docker-compose/scheduler.json
@@ -1,17 +1,18 @@
 {
-  "stores": {
-    "GRPC_LOCAL_STORE": {
+  "stores": [
+    {
       // Note: This file is used to test GRPC store.
+      "name": "GRPC_LOCAL_STORE",
       "grpc": {
         "instance_name": "main",
         "endpoints": [
           {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"}
         ],
         "store_type": "cas"
       }
-    },
-    "GRPC_LOCAL_AC_STORE": {
+    }, {
       // Note: This file is used to test GRPC store.
+      "name": "GRPC_LOCAL_AC_STORE",
       "grpc": {
         "instance_name": "main",
         "endpoints": [
@@ -20,18 +21,19 @@
         "store_type": "ac"
       }
     }
-  },
-  "schedulers": {
-    "MAIN_SCHEDULER": {
-      "simple": {
+  ],
+  "schedulers": [
+    {
+      "name": "MAIN_SCHEDULER",
+      "siple": {
         "supported_platform_properties": {
           "cpu_count": "minimum",
           "OSFamily": "priority",
           "container-image": "priority"
         }
       }
     }
-  },
+  ],
   "servers": [{
     "listener": {
       "http": {

diff --git a/deployment-examples/docker-compose/worker.json b/deployment-examples/docker-compose/worker.json
@@ -1,45 +1,47 @@
 {
-  "stores": {
-    "GRPC_LOCAL_STORE": {
+  "stores": [
+    {
       // Note: This file is used to test GRPC store.
+      "name": "GRPC_LOCAL_STORE",
       "grpc": {
         "instance_name": "main",
         "endpoints": [
           {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"}
         ],
         "store_type": "cas"
       }
-    },
-    "GRPC_LOCAL_AC_STORE": {
+    }, {
       // Note: This file is used to test GRPC store.
+      "name": "GRPC_LOCAL_AC_STORE",
       "grpc": {
         "instance_name": "main",
         "endpoints": [
           {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"}
         ],
         "store_type": "ac"
       }
-    },
-    "WORKER_FAST_SLOW_STORE": {
-      "fast_slow": {
+    }, {
+      "name": "WORKER_FAST_SLOW_STORE",
+      "fastSlow": {
         "fast": {
+          "name": "fs-worker",
           "filesystem": {
             "content_path": "/root/.cache/nativelink/data-worker-test/content_path-cas",
             "temp_path": "/root/.cache/nativelink/data-worker-test/tmp_path-cas",
             "eviction_policy": {
-              // 10gb.
-              "max_bytes": 10000000000,
+              "max_bytes": "10Gb",
             }
           }
         },
         "slow": {
-          "ref_store": {
+          "name": "ref-worker",
+          "ref": {
             "name": "GRPC_LOCAL_STORE",
           }
         }
       }
     }
-  },
+  ],
   "workers": [{
     "local": {
       "worker_api_endpoint": {

diff --git a/flake.nix b/flake.nix
@@ -203,6 +203,14 @@
               cargoArtifacts = cargoArtifactsFor p;
             });
 
+        nativelinkControllerFor = p:
+          (craneLibFor p).buildPackage ((commonArgsFor p)
+            // {
+              pname = "nativelink-controller";
+              cargoExtraArgs = "--package=nativelink-controller";
+              cargoArtifacts = cargoArtifactsFor p;
+            });
+
         nativeTargetPkgs =
           if pkgs.system == "x86_64-linux"
           then pkgs.pkgsCross.musl64
@@ -225,6 +233,14 @@
             cargoExtraArgs = "--features enable_tokio_console";
           });
 
+        # TODO(aaronmondal): Enable.
+        # nativelink-controller = nativelinkControllerFor nativeTargetPkgs;
+
+        # These two can be built by all build platforms. This is not true for
+        # darwin targets which are only buildable via native compilation.
+        nativelink-controller-aarch64-linux = nativelinkControllerFor pkgs.pkgsCross.aarch64-multiplatform-musl;
+        nativelink-controller-x86_64-linux = nativelinkControllerFor pkgs.pkgsCross.musl64;
+
         publish-ghcr = pkgs.callPackage ./tools/publish-ghcr.nix {};
 
         local-image-test = pkgs.callPackage ./tools/local-image-test.nix {};
@@ -279,6 +295,35 @@
             };
           };
 
+        nativelink-controller-image = let
+          nativelinkControllerForImage =
+            if pkgs.stdenv.isx86_64
+            then nativelink-controller-x86_64-linux
+            else nativelink-controller-aarch64-linux;
+        in
+          buildImage {
+            name = "nativelink-controller";
+            copyToRoot = [
+              (pkgs.buildEnv {
+                name = "nativelink-buildEnv";
+                paths = [nativelinkControllerForImage];
+                pathsToLink = ["/bin"];
+              })
+            ];
+            config = {
+              Entrypoint = [(pkgs.lib.getExe' nativelinkControllerForImage "nativelink-controller")];
+              Labels = {
+                "org.opencontainers.image.description" = "Controller for NativeLink CRDs.";
+                "org.opencontainers.image.documentation" = "https://github.com/TraceMachina/nativelink";
+                "org.opencontainers.image.licenses" = "Apache-2.0";
+                "org.opencontainers.image.revision" = "${self.rev or self.dirtyRev or "dirty"}";
+                "org.opencontainers.image.source" = "https://github.com/TraceMachina/nativelink";
+                "org.opencontainers.image.title" = "NativeLink Controller";
+                "org.opencontainers.image.vendor" = "Trace Machina, Inc.";
+              };
+            };
+          };
+
         nativelink-worker-init = pkgs.callPackage ./tools/nativelink-worker-init.nix {inherit buildImage self nativelink-image;};
 
         rbe-autogen = pkgs.callPackage ./local-remote-execution/rbe-autogen.nix {
@@ -415,6 +460,7 @@
               nativelink-aarch64-linux
               nativelink-debug
               nativelink-image
+              nativelink-controller-image
               nativelink-is-executable-test
               nativelink-worker-init
               nativelink-x86_64-linux
@@ -513,6 +559,8 @@
               pkgs.fluxcd
               pkgs.go
               pkgs.kustomize
+              pkgs.grpcurl
+              pkgs.kubectx
 
               ## Web
               pkgs.bun # got patched to the newest version (v.1.1.25)

diff --git a/kubernetes/configmaps/cas.json b/kubernetes/configmaps/cas.json
@@ -2,48 +2,51 @@
 // `~/.cache/nativelink`. When this location is mounted as a PersistentVolume
 // it persists the cache across restarts.
 {
-  "stores": {
-    "CAS_MAIN_STORE": {
-      "existence_cache": {
+  "stores": [
+    {
+      "name": "CAS_MAIN_STORE",
+      "existenceCache": {
         "backend": {
+          "name": "compression-store",
           "compression": {
             "compression_algorithm": {
               "lz4": {}
             },
             "backend": {
+              "name": "fs-cas",
               "filesystem": {
                 "content_path": "~/.cache/nativelink/content_path-cas",
                 "temp_path": "~/.cache/nativelink/tmp_path-cas",
                 "eviction_policy": {
-                  // 10gb.
-                  "max_bytes": 10000000000,
+                  "max_bytes": "10Gb"
                 }
               }
             }
           }
         }
       }
-    },
-    "AC_MAIN_STORE": {
-      "completeness_checking": {
+    }, {
+      "name": "AC_MAIN_STORE",
+      "completenessChecking": {
         "backend": {
+          "name": "fs-ac",
           "filesystem": {
             "content_path": "~/.cache/nativelink/content_path-ac",
             "temp_path": "~/.cache/nativelink/tmp_path-ac",
             "eviction_policy": {
-              // 500mb.
-              "max_bytes": 500000000,
+              "max_bytes": "500Mb",
             }
           }
         },
         "cas_store": {
-          "ref_store": {
+          "name": "ref-cas",
+          "ref": {
             "name": "CAS_MAIN_STORE"
           }
         }
       }
     }
-  },
+  ],
   "servers": [{
     "listener": {
       "http": {

diff --git a/kubernetes/configmaps/scheduler.json b/kubernetes/configmaps/scheduler.json
@@ -1,17 +1,18 @@
 {
-  "stores": {
-    "GRPC_LOCAL_STORE": {
+  "stores": [
+    {
       // Note: This file is used to test GRPC store.
+      "name": "GRPC_LOCAL_STORE",
       "grpc": {
         "instance_name": "main",
         "endpoints": [
           {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"}
         ],
         "store_type": "cas"
       }
-    },
-    "GRPC_LOCAL_AC_STORE": {
+    }, {
       // Note: This file is used to test GRPC store.
+      "name": "GRPC_LOCAL_AC_STORE",
       "grpc": {
         "instance_name": "main",
         "endpoints": [
@@ -20,11 +21,12 @@
         "store_type": "ac"
       }
     }
-  },
-  "schedulers": {
-    "MAIN_SCHEDULER": {
+  ],
+  "schedulers": [
+    {
       // TODO(adams): use the right scheduler because reclient doesn't use the cached results?
       // TODO(adams): max_bytes_per_stream
+      "name": "MAIN_SCHEDULER",
       "simple": {
         "supported_platform_properties": {
           "cpu_count": "priority",
@@ -46,7 +48,7 @@
         }
       }
     }
-  },
+  ],
   "servers": [{
     "listener": {
       "http": {