diff --git a/.gitignore b/.gitignore index 8bccc34ffc0..8083d3f1ee8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ .zebra-state/ # Nix configs shell.nix +# Nix builds +/result # Docker compose env files *.env diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000000..cd4b0b52100 --- /dev/null +++ b/flake.lock @@ -0,0 +1,179 @@ +{ + "nodes": { + "advisory-db": { + "flake": false, + "locked": { + "lastModified": 1732819720, + "narHash": "sha256-6H7mKBKw3VErpGcCGEamBYJsopvqqdFmJhl8slfCtOQ=", + "owner": "rustsec", + "repo": "advisory-db", + "rev": "9dc4a0bb102451e3c71e1b639068aec5a3e1f5f3", + "type": "github" + }, + "original": { + "owner": "rustsec", + "repo": "advisory-db", + "type": "github" + } + }, + "crane": { + "locked": { + "lastModified": 1732990152, + "narHash": "sha256-brVP7vwtlMZCesoA5K+JWDH1MQlzGEp5PWs3DIhtWhY=", + "owner": "ipetkov", + "repo": "crane", + "rev": "a031189a409b41826c7b767a84f2cff60152e5b5", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "fenix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rust-analyzer-src": "rust-analyzer-src" + }, + "locked": { + "lastModified": 1732689334, + "narHash": "sha256-yKI1KiZ0+bvDvfPTQ1ZT3oP/nIu3jPYm4dnbRd6hYg4=", + "owner": "nix-community", + "repo": "fenix", + "rev": "a8a983027ca02b363dfc82fbe3f7d9548a8d3dce", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": [ + "systems" + ] + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "home-manager": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1726989464, + "narHash": "sha256-Vl+WVTJwutXkimwGprnEtXc/s/s8sMuXzqXaspIGlwM=", + "owner": "nix-community", + "repo": "home-manager", + "rev": "2f23fa308a7c067e52dfcc30a0758f47043ec176", + "type": "github" + }, + "original": { + "owner": "nix-community", + "ref": "release-24.05", + "repo": "home-manager", + "type": "github" + } + }, + "nix-darwin": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1732603785, + "narHash": "sha256-AEjWTJwOmSnVYsSJCojKgoguGfFfwel6z/6ud6UFMU8=", + "owner": "LnL7", + "repo": "nix-darwin", + "rev": "6ab87b7c84d4ee873e937108c4ff80c015a40c7a", + "type": "github" + }, + "original": { + "owner": "LnL7", + "repo": "nix-darwin", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1732967653, + "narHash": "sha256-FFE2Ac749LNALTR9toJCflRBenkR6Cc5G165pqgwg/0=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "d634159c8bf583a28a7d963b9a3cb6729bb3b2f1", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "release-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "advisory-db": "advisory-db", + "crane": "crane", + "fenix": "fenix", + "flake-utils": "flake-utils", + "home-manager": "home-manager", + "nix-darwin": "nix-darwin", + "nixpkgs": "nixpkgs", + "systems": "systems" + } + }, + "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1732633904, + "narHash": "sha256-7VKcoLug9nbAN2txqVksWHHJplqK9Ou8dXjIZAIYSGc=", + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "8d5e91c94f80c257ce6dbdfba7bd63a5e8a03fa6", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000000..5aea2843c4d --- /dev/null +++ b/flake.nix @@ -0,0 +1,192 @@ +{ + description = '' + Zebra: the Zcash Foundation's independent, consensus-compatible implementation of a Zcash node + ''; + + nixConfig = { + extra-experimental-features = ["no-url-literals"]; + extra-substituters = [ + "https://cache.garnix.io" # To benefit from the CI results + "https://nix-community.cachix.org" # Contains rust toolchains + ]; + extra-trusted-public-keys = [ + "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g=" + "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + ]; + use-registries = false; + sandbox = "relaxed"; + }; + + outputs = { + advisory-db, + crane, + fenix, + flake-utils, + home-manager, + nix-darwin, + nixpkgs, + self, + systems, + }: let + ## Since they currently line up, we just use `nix-systems/default`, but see + ## https://zebra.zfnd.org/user/supported-platforms.html for official support tiers. + ## + ## NixOS is not a supported OS, and this flake is not a supported distribution of Zebra, but + ## here’s an approximation of what you can expect from this flake: + ## - aarch64-darwin – tier 3 + ## - aarch64-linux (on Debian 11) – tier 3 + ## - x86_64-darwin – tier 2 + ## - x86_64-linux + ## - on Debian 11 – tier 1 + ## - on Ubuntu “latest” – tier 2 + supportedSystems = import systems; + + lib = import ./nix/lib { + inherit crane; + inherit (nixpkgs) lib; + }; + + src = nixpkgs.lib.cleanSourceWith { + src = ./.; + filter = path: type: + nixpkgs.lib.foldr + (e: acc: nixpkgs.lib.hasSuffix e path || acc) + true [".bin" ".proto" ".txt" ".utf8" ".vk"] + || lib.crane.filterCargoSources path type; + }; + + ## This sets up some common attributes that _should_ be set in the workspace Cargo.toml, but + ## aren’t. + workspace = + lib.crane.crateNameFromCargoToml {cargoToml = ./zebrad/Cargo.toml;} // {pname = "zebra";}; + + mkCraneLib = pkgs: + (crane.mkLib pkgs).overrideToolchain (import ./nix/rust-toolchain.nix {inherit fenix pkgs;}); + + localPackages = { + pkgs, + craneLib ? mkCraneLib pkgs, + }: + import ./nix/packages { + inherit craneLib src workspace; + inherit (pkgs) callPackage; + }; + in + { + overlays.default = final: prev: localPackages {pkgs = final;}; + + darwinModules = { + default = self.darwinModules.zebra; + zebra = import ./nix/modules/zebra/darwin.nix; + }; + homeModules = { + default = self.homeModules.zebra; + zebra = import ./nix/modules/zebra/home.nix; + }; + nixosModules = { + default = self.nixosModules.zebra; + zebra = import ./nix/modules/zebra/nixos.nix; + }; + + ## For testing the `overlays` and `darwinModules` as well as provide an example configuration + ## for users. + darwinConfigurations = builtins.listToAttrs (map (system: { + name = "${system}-example"; + value = import ./nix/darwin-configuration.nix { + inherit nix-darwin nixpkgs system; + zebra = self; + }; + }) + (builtins.filter (nixpkgs.lib.hasSuffix "-darwin") supportedSystems)); + + ## For testing the `overlays` and `homeModules` as well as provide an example configuration + ## for users. + homeConfigurations = builtins.listToAttrs (map (system: { + name = "${system}-example"; + value = import ./nix/home-configuration.nix { + inherit home-manager nixpkgs system; + zebra = self; + }; + }) + supportedSystems); + + ## For testing the `overlays` and `nixosModules` as well as provide an example configuration + ## for users. + nixosConfigurations = builtins.listToAttrs (map (system: { + name = "${system}-example"; + value = import ./nix/nixos-configuration.nix { + inherit nixpkgs system; + zebra = self; + }; + }) + (builtins.filter (nixpkgs.lib.hasSuffix "-linux") supportedSystems)); + } + // flake-utils.lib.eachSystem supportedSystems (system: let + pkgs = nixpkgs.legacyPackages.${system}; + + craneLib = mkCraneLib pkgs; + + packages = localPackages {inherit craneLib pkgs;}; + + cargoArtifacts = packages.zebra-deps; + in { + apps = + {default = self.apps.${system}.zebrad;} + // (import ./nix/apps.nix { + inherit flake-utils; + drv = self.packages.${system}.zebra; + }); + + packages = + {default = self.packages.${system}.zebra;} + // packages; + + devShells.default = craneLib.devShell { + inherit cargoArtifacts; + checks = self.checks.${system}; + inputsFrom = builtins.attrValues self.packages.${system}; + packages = [pkgs.home-manager]; + LIBCLANG_PATH = pkgs.libclang.lib + "/lib"; + }; + + checks = import ./nix/checks.nix { + inherit advisory-db cargoArtifacts craneLib pkgs src workspace; + inherit (nixpkgs) lib; + }; + + formatter = pkgs.alejandra; + }); + + inputs = { + advisory-db = { + flake = false; + url = "github:rustsec/advisory-db"; + }; + + crane.url = "github:ipetkov/crane"; + + fenix = { + inputs.nixpkgs.follows = "nixpkgs"; + url = "github:nix-community/fenix"; + }; + + flake-utils = { + inputs.systems.follows = "systems"; + url = "github:numtide/flake-utils"; + }; + + home-manager = { + inputs.nixpkgs.follows = "nixpkgs"; + url = "github:nix-community/home-manager/release-24.05"; + }; + + nix-darwin = { + inputs.nixpkgs.follows = "nixpkgs"; + url = "github:LnL7/nix-darwin"; + }; + + nixpkgs.url = "github:NixOS/nixpkgs/release-24.05"; + + systems.url = "github:nix-systems/default"; + }; +} diff --git a/garnix.yaml b/garnix.yaml new file mode 100644 index 00000000000..9d4a3e13531 --- /dev/null +++ b/garnix.yaml @@ -0,0 +1,7 @@ +builds: + exclude: + # This system isn’t actually supported by garnix (see garnix-io/issues#94) + - "*.x86_64-darwin-example" + include: + - '*.*' + - '*.*.*' diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 00000000000..c250ffccc16 --- /dev/null +++ b/nix/README.md @@ -0,0 +1,94 @@ +# Zebra & Nix + +This exposes a `zebrad` package in various ways, but it’s recommended to lean on one of the higher-level modules to integrate Zebra with your system. + +## modules + +### Home Manager + +Zebra is generally intended to be run as a user service, so Home Manager tends to make the most sense for configuring it. + +Home Manager can manage keeping your zebrad instance up and running with either systemd (Linux) or launchd (MacOS). + +You first need to make this repository available to Home Manager. Here is a minimal flake for this purpose: + +```nix +{ + inputs = { + home-manager.url = "github:nix-community/home-manager/release-24.05"; + nixpkgs.url = "github:NixOS/nixpkgs/release-24.05"; + zebra.url = "github:sellout/zebra/nix-flake"; + }; + + outputs = {home-manager, nixpkgs, self, zebra}: { + homeConfigurations."@" = import ./home-configuration.nix { + inherit home-manager nixpkgs zebra; + system = "x86_64-linux"; + }; + }; +} +``` + +See [home-configuration.nix](./home-configuration.nix) for how to connect Zebra to your configuration and [our test configuration](./modules/home.nix) for an example of how to set up Zebra. + +### NixOS + +Similarly, this can be run as a NixOS user service. + +``` nix +{ + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/release-24.05"; + zebra.url = "github:sellout/zebra/nix-flake"; + }; + + outputs = {home-manager, nixpkgs, self, zebra}: { + nixosConfigurations."" = import ./nixos-configuration.nix { + inherit nixpkgs zebra; + system = "x86_64-linux"; + }; + }; +} +``` + +### nix-darwin + +Or as a nix-darwin user service. + +```nix +{ + inputs = { + nix-darwin.url = "github:LnL7/nix-darwin"; + nixpkgs.url = "github:NixOS/nixpkgs/release-24.05"; + zebra.url = "github:sellout/zebra/nix-flake"; + }; + + outputs = {nix-darwin, nixpkgs, self, zebra}: { + darwinConfigurations."" = import ./darwin-configuration.nix { + inherit nix-darwin nixpkgs zebra; + system = "x86_64-linux"; + }; + }; +} +``` + +## overlay + +If you just want to access the applications directly, you can get them via the default overlay: + +```nix +let + newPkgs = pkgs.appendOverlays [zebra.overlays.default]; +in [ + newPkgs.zebra-scanner + newPkgs.zebrad +] +``` + +## execution + +They can also be run directly, without installing anything. + +```bash +nix run github:sellout/zebra/nix-flake#zebrad -- start +``` diff --git a/nix/apps.nix b/nix/apps.nix new file mode 100644 index 00000000000..aaf9976a4ee --- /dev/null +++ b/nix/apps.nix @@ -0,0 +1,15 @@ +{ + drv, + flake-utils, +}: let + mkNamedApp = name: { + inherit name; + value = flake-utils.lib.mkApp {inherit drv name;}; + }; +in + ## There are additional apps in the workspace, but they are only produced when particular + ## Cargo features are enabled. + builtins.listToAttrs (map mkNamedApp [ + "zebra-scanner" + "zebrad" + ]) diff --git a/nix/checks.nix b/nix/checks.nix new file mode 100644 index 00000000000..2fcdf5c92c1 --- /dev/null +++ b/nix/checks.nix @@ -0,0 +1,47 @@ +{ + advisory-db, + cargoArtifacts, + craneLib, + lib, + pkgs, + src, + workspace, +}: { + audit = craneLib.cargoAudit {inherit advisory-db src;}; + + clippy = craneLib.cargoClippy (workspace + // { + inherit cargoArtifacts src; + buildInputs = lib.optionals pkgs.stdenv.hostPlatform.isDarwin [ + pkgs.darwin.apple_sdk.frameworks.Security + pkgs.libiconv + ]; + nativeBuildInputs = [pkgs.protobuf]; + stdenv = pkgs.clangStdenv; + LIBCLANG_PATH = pkgs.libclang.lib + "/lib"; + cargoClippyExtraArgs = "--all-targets"; # -- --deny warnings"; + }); + + deny = craneLib.cargoDeny (workspace + // { + inherit src; + cargoDenyChecks = lib.concatStringsSep " " ["bans" "sources"]; + }); + + doc = craneLib.cargoDoc (workspace + // { + inherit cargoArtifacts src; + buildInputs = lib.optionals pkgs.stdenv.hostPlatform.isDarwin [ + pkgs.darwin.apple_sdk.frameworks.Security + pkgs.libiconv + ]; + nativeBuildInputs = [pkgs.protobuf]; + stdenv = pkgs.clangStdenv; + LIBCLANG_PATH = pkgs.libclang.lib + "/lib"; + }); + + fmt = craneLib.cargoFmt (workspace // {inherit src;}); + + # toml-fmt = + # craneLib.taploFmt (workspace // {src = lib.sources.sourceFilesBySuffices src [".toml"];}); +} diff --git a/nix/darwin-configuration.nix b/nix/darwin-configuration.nix new file mode 100644 index 00000000000..8888b000bc2 --- /dev/null +++ b/nix/darwin-configuration.nix @@ -0,0 +1,16 @@ +{ + nix-darwin, + nixpkgs, + system, + zebra, +}: +nix-darwin.lib.darwinSystem { + pkgs = import nixpkgs { + inherit system; + overlays = [zebra.overlays.default]; # ← Makes `zebra` package available + }; + modules = [ + zebra.darwinModules.default # ← Makes `services.zebra` options available + ./modules/darwin-configuration.nix # ← This file contains your Zebra configuration + ]; +} diff --git a/nix/home-configuration.nix b/nix/home-configuration.nix new file mode 100644 index 00000000000..efe68f39b4f --- /dev/null +++ b/nix/home-configuration.nix @@ -0,0 +1,14 @@ +{ + home-manager, + nixpkgs, + system, + zebra, +}: +home-manager.lib.homeManagerConfiguration { + modules = [ + {nixpkgs.overlays = [zebra.overlays.default];} # ← Makes `zebra` package available + zebra.homeModules.default # ← Makes `services.zebra` options available + ./modules/home.nix # ← This file contains your Zebra configuration + ]; + pkgs = nixpkgs.legacyPackages.${system}; +} diff --git a/nix/lib/crane.nix b/nix/lib/crane.nix new file mode 100644 index 00000000000..35d7dd61ea6 --- /dev/null +++ b/nix/lib/crane.nix @@ -0,0 +1,14 @@ +## TODO: This extracts various functions from the crane repo, bypassing `crane.mkLib`, since these +## don’t require `pkgs`. (See ipetkov/crane#699) +{ + crane, + lib, +}: let + internalCrateNameFromCargoToml = + import "${crane}/lib/internalCrateNameFromCargoToml.nix" {inherit lib;}; +in { + crateNameFromCargoToml = + import "${crane}/lib/crateNameFromCargoToml.nix" {inherit internalCrateNameFromCargoToml lib;}; + + filterCargoSources = import "${crane}/lib/filterCargoSources.nix" {inherit lib;}; +} diff --git a/nix/lib/default.nix b/nix/lib/default.nix new file mode 100644 index 00000000000..d0ce7fa73e4 --- /dev/null +++ b/nix/lib/default.nix @@ -0,0 +1,24 @@ +{ + crane, + lib, +}: { + crane = import ./crane.nix {inherit crane lib;}; + + ## Accepts separate configurations for nix-darwin, Home Manager, and NixOS, returning the correct + ## one for whichever configuration is being built (guessing based on the attributes defined by + ## `options`). + ## + ## This is useful for writing modules that work across multiple types of configuration. + multiConfig = options: { + darwin ? {}, + home ? {}, + nixos ? {}, + }: { + config = + if options ? homebrew + then darwin + else if options ? home + then home + else nixos; + }; +} diff --git a/nix/modules/configuration.nix b/nix/modules/configuration.nix new file mode 100644 index 00000000000..482c5ea2a5f --- /dev/null +++ b/nix/modules/configuration.nix @@ -0,0 +1,25 @@ +{ + services.zebra = { + ## If this is false, Zebra will not run, no matter what else is set in this section. + enable = true; + ## This exactly matches the structure of zebrad.toml, just using Nix syntax instead of TOML. + config = { + network.network = "Testnet"; + state.cache_dir = "/var/cache/zebrad"; + tracing = { + ## This automatically enables the flamegraph support in Zebra. + flamegraph = "/var/lib/zebrad/flamegraph"; + use_color = true; + }; + }; + ## Other features to enable in the Zebra package. There is no need to list things that are + ## required by entries in `services.zebra.config`, as those get enabled as needed (although + ## adding them can improve Nix cache hits, since different configs on different machines won’t + ## require different builds). + extraFeatures = ["elasticsearch"]; + }; + + boot.loader.grub.devices = ["/dev/sda1"]; + fileSystems."/".device = "/dev/sda1"; + system.stateVersion = "24.05"; +} diff --git a/nix/modules/darwin-configuration.nix b/nix/modules/darwin-configuration.nix new file mode 100644 index 00000000000..be5f0f625f4 --- /dev/null +++ b/nix/modules/darwin-configuration.nix @@ -0,0 +1,23 @@ +{ + services.zebra = { + ## If this is false, Zebra will not run, no matter what else is set in this section. + enable = true; + ## This exactly matches the structure of zebrad.toml, just using Nix syntax instead of TOML. + config = { + network.network = "Testnet"; + state.cache_dir = "/var/cache/zebrad"; + tracing = { + ## This automatically enables the flamegraph support in Zebra. + flamegraph = "/var/lib/zebrad/flamegraph"; + use_color = true; + }; + }; + ## Other features to enable in the Zebra package. There is no need to list things that are + ## required by entries in `services.zebra.config`, as those get enabled as needed (although + ## adding them can improve Nix cache hits, since different configs on different machines won’t + ## require different builds). + extraFeatures = ["elasticsearch"]; + }; + + system.stateVersion = 5; +} diff --git a/nix/modules/home.nix b/nix/modules/home.nix new file mode 100644 index 00000000000..642ef043594 --- /dev/null +++ b/nix/modules/home.nix @@ -0,0 +1,28 @@ +{config, ...}: { + services.zebra = { + ## If this is false, Zebra will not run, no matter what else is set in this section. + enable = true; + ## This exactly matches the structure of zebrad.toml, just using Nix syntax instead of TOML. + config = { + network.network = "Testnet"; + state.cache_dir = "${config.xdg.cacheHome}/zebrad"; + tracing = { + ## This automatically enables the flamegraph Cargo feature in Zebra. + flamegraph = "${config.xdg.stateHome}/zebrad/flamegraph"; + use_color = true; + }; + }; + ## Other Cargo features to enable in the Zebra package. There is no need to list things that are + ## required by entries in `services.zebra.config`, as those get enabled as needed (although + ## adding them can improve Nix cache hits, since different configs on different machines won’t + ## require different builds). + extraFeatures = ["elasticsearch"]; + }; + + ## These attributes are simply required by Home Manager. + home = { + homeDirectory = /tmp/example; + stateVersion = "24.05"; + username = "example-user"; + }; +} diff --git a/nix/modules/zebra/darwin.nix b/nix/modules/zebra/darwin.nix new file mode 100644 index 00000000000..debffaf89da --- /dev/null +++ b/nix/modules/zebra/darwin.nix @@ -0,0 +1,27 @@ +{ + config, + lib, + options, + pkgs, + ... +}: let + cfg = config.services.zebra; +in { + imports = [./generic.nix]; + + config = lib.mkIf cfg.enable (let + toml = pkgs.formats.toml {}; + + configFile = toml.generate "zebrad.toml" cfg.config; + in { + environment = { + etc."zebrad/zebrad.toml".source = configFile; + systemPackages = [cfg.package]; + }; + + launchd.agents.zebrad.serviceConfig = import ./launchd.nix { + inherit configFile lib; + inherit (cfg) package; + }; + }); +} diff --git a/nix/modules/zebra/generic.nix b/nix/modules/zebra/generic.nix new file mode 100644 index 00000000000..d6795d25585 --- /dev/null +++ b/nix/modules/zebra/generic.nix @@ -0,0 +1,65 @@ +{ + config, + lib, + options, + pkgs, + ... +}: let + cfg = config.services.zebra; +in { + options.services.zebra = { + enable = lib.mkEnableOption "Zebra"; + + basePackage = lib.mkPackageOption pkgs "Zebra" {default = ["zebra"];}; + + package = lib.mkOption { + type = lib.types.package; + readOnly = true; + description = '' + The _actual_ package, built from the base package with `extraFeatures` applied. + ''; + }; + + config = lib.mkOption { + type = lib.types.attrs; + default = { + tracing.use_journald = pkgs.stdenv.hostPlatform.isLinux; + }; + defaultText = lib.literalExpression '' + { + tracing.use_journald = pkgs.stdenv.hostPlatform.isLinux; + } + ''; + description = "A Nix attribute set of the values to put in zebrad.toml"; + example = { + tracing.progress_bar = "summary"; + }; + }; + + extraFeatures = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = []; + description = '' + A list of optional features to enable in the package. See + https://docs.rs/zebrad/latest/zebrad/index.html#zebra-feature-flags + for the avaliable values. + + __NB__: Some features are enabled automatically if config options that require them are set. + For example, `services.zebra.config.tracing.use_journald = true` will cause + `"journald"` to be enabled and + `services.zebra.config.tracing.flamegraph = "some/path"` will cause `"flamegraph"` to + be enabled. + ''; + example = ["elasticsearch" "journald" "prometheus"]; + }; + }; + + config = lib.mkIf cfg.enable (let + combinedFeatures = + cfg.extraFeatures + ++ lib.optional (cfg.config.tracing.flamegraph or null != null) "flamegraph" + ++ lib.optional (cfg.config.tracing.use_journald or false) "journald"; + in { + services.zebra.package = cfg.basePackage.override {extraFeatures = combinedFeatures;}; + }); +} diff --git a/nix/modules/zebra/home.nix b/nix/modules/zebra/home.nix new file mode 100644 index 00000000000..983775e9e80 --- /dev/null +++ b/nix/modules/zebra/home.nix @@ -0,0 +1,35 @@ +{ + config, + lib, + options, + pkgs, + ... +}: let + cfg = config.services.zebra; +in { + imports = [./generic.nix]; + + config = lib.mkIf cfg.enable (let + toml = pkgs.formats.toml {}; + + configFile = toml.generate "zebrad.toml" cfg.config; + in { + home.packages = [cfg.package]; + + launchd.agents.zebra = { + enable = true; + config = import ./launchd.nix { + inherit configFile lib; + inherit (cfg) package; + }; + }; + + systemd.user.services.zebrad = import ./systemd.nix { + inherit configFile lib; + inherit (cfg) package; + }; + + ## TODO: Maybe need to put this somewhere else on darwin. + xdg.configFile."zebrad.toml".source = configFile; + }); +} diff --git a/nix/modules/zebra/launchd.nix b/nix/modules/zebra/launchd.nix new file mode 100644 index 00000000000..61317eff69c --- /dev/null +++ b/nix/modules/zebra/launchd.nix @@ -0,0 +1,11 @@ +{ + configFile, + lib, + package, +}: { + Program = lib.getExe package; + ProgramArguments = ["--config" (toString configFile) "start"]; + ProcessType = "Background"; + KeepAlive = true; + RunAtLoad = true; +} diff --git a/nix/modules/zebra/nixos.nix b/nix/modules/zebra/nixos.nix new file mode 100644 index 00000000000..57af06dc13f --- /dev/null +++ b/nix/modules/zebra/nixos.nix @@ -0,0 +1,33 @@ +{ + config, + lib, + options, + pkgs, + ... +}: let + cfg = config.services.zebra; +in { + imports = [./generic.nix]; + + config = lib.mkIf cfg.enable (let + toml = pkgs.formats.toml {}; + + configFile = toml.generate "zebrad.toml" cfg.config; + in { + environment = { + etc."zebrad/zebrad.toml".source = configFile; + systemPackages = [cfg.package]; + }; + + systemd.services.zebrad = let + config = import ./systemd.nix { + inherit configFile lib; + inherit (cfg) package; + }; + in { + unitConfig = config.Unit; + serviceConfig = config.Service; + wantedBy = config.Install.WantedBy; + }; + }); +} diff --git a/nix/modules/zebra/systemd.nix b/nix/modules/zebra/systemd.nix new file mode 100644 index 00000000000..bc978f0375b --- /dev/null +++ b/nix/modules/zebra/systemd.nix @@ -0,0 +1,22 @@ +## TODO: This should instead read zebrad/systemd/zebrad.service, and modify the +## few things that need to be. +{ + configFile, + lib, + package, +}: let + programPath = lib.getExe package; +in { + Unit.AssertPathExists = programPath; + + Service = { + ExecStart = "${programPath} --config ${lib.escapeShellArg configFile} start"; + Restart = "always"; + PrivateTmp = true; + NoNewPrivileges = true; + StandardOutput = "journal"; + StandardError = "journal"; + }; + + Install.WantedBy = ["default.target"]; +} diff --git a/nix/nixos-configuration.nix b/nix/nixos-configuration.nix new file mode 100644 index 00000000000..cec865c3311 --- /dev/null +++ b/nix/nixos-configuration.nix @@ -0,0 +1,13 @@ +{ + nixpkgs, + system, + zebra, +}: +nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + {nixpkgs.overlays = [zebra.overlays.default];} # ← Makes `zebra` package available + zebra.nixosModules.default # ← Makes `services.zebra` options available + ./modules/configuration.nix # ← This file contains your Zebra configuration + ]; +} diff --git a/nix/packages/default.nix b/nix/packages/default.nix new file mode 100644 index 00000000000..df0f61f693d --- /dev/null +++ b/nix/packages/default.nix @@ -0,0 +1,14 @@ +{ + callPackage, + craneLib, + src, + workspace, +}: let + zebra-deps = callPackage ./zebra-deps.nix {inherit craneLib src workspace;}; +in { + inherit zebra-deps; + zebra = callPackage ./zebra { + inherit craneLib src workspace; + cargoArtifacts = zebra-deps; + }; +} diff --git a/nix/packages/zebra-deps.nix b/nix/packages/zebra-deps.nix new file mode 100644 index 00000000000..d6bee377f66 --- /dev/null +++ b/nix/packages/zebra-deps.nix @@ -0,0 +1,22 @@ +{ + clangStdenv, + craneLib, + lib, + libclang, + libiconv, + src, + stdenv, + workspace, +}: +craneLib.buildDepsOnly ( + workspace + // { + inherit src; + + buildInputs = lib.optional stdenv.hostPlatform.isDarwin libiconv; + + stdenv = clangStdenv; + + LIBCLANG_PATH = libclang.lib + "/lib"; + } +) diff --git a/nix/packages/zebra/default.nix b/nix/packages/zebra/default.nix new file mode 100644 index 00000000000..147ae562509 --- /dev/null +++ b/nix/packages/zebra/default.nix @@ -0,0 +1,57 @@ +{ + cargoArtifacts, + clangStdenv, + craneLib, + darwin, + lib, + libclang, + libiconv, + protobuf, + src, + stdenv, + workspace, + ## Any additional features to compile zebrad with. See + ## https://docs.rs/zebrad/latest/zebrad/index.html#zebra-feature-flags for available features. + extraFeatures ? [], + ## Whether to use exactly the dependency versions specified in the Cargo.lock file. + locked ? true, +}: +craneLib.buildPackage ( + workspace + // { + inherit cargoArtifacts src; + + strictDeps = true; + + buildInputs = + if stdenv.hostPlatform.isDarwin + then [ + darwin.apple_sdk.frameworks.Security + darwin.apple_sdk.frameworks.SystemConfiguration + libiconv + ] + else []; + + nativeBuildInputs = [protobuf]; + + stdenv = clangStdenv; + + LIBCLANG_PATH = libclang.lib + "/lib"; + + cargoExtraArgs = lib.escapeShellArgs ( + ["--features" (lib.concatStringsSep "," extraFeatures)] + ++ lib.optional locked "--locked" + ); + + ## TODO: Use the fixed-output derivation trick to allow network access during tests, so this can + ## be removed. + ZEBRA_SKIP_NETWORK_TESTS = true; + + cargoTestExtraArgs = let + failingTests = import ./failing-tests.nix {inherit extraFeatures lib stdenv;}; + in + lib.escapeShellArgs (["--"] ++ lib.concatMap (test: ["--skip" test]) failingTests); + + meta.mainProgram = "zebrad"; + } +) diff --git a/nix/packages/zebra/failing-tests.nix b/nix/packages/zebra/failing-tests.nix new file mode 100644 index 00000000000..1228e540aa1 --- /dev/null +++ b/nix/packages/zebra/failing-tests.nix @@ -0,0 +1,118 @@ +### Tests that fail on Nix, but aren’t skipped by `ZEBRA_SKIP_NETWORK_TESTS`. +{ + extraFeatures, + lib, + stdenv, +}: +## NB: On Darwin, `__darwinAllowLocalNetworking` would allow many of these tests to pass, but only +## in a non-sandboxed environment. +[ + ## zebra-scan + "scan_binary_starts" + ## zebrad – acceptance + "config_tests" + "end_of_support_is_checked_at_start" + "ephemeral_existing_directory" + "ephemeral_missing_directory" + "external_address" + "misconfigured_ephemeral_existing_directory" + "misconfigured_ephemeral_missing_directory" + "persistent_mode" + "trusted_chain_sync_handles_forks_correctly" +] +++ lib.optionals stdenv.hostPlatform.isDarwin [ + ## zebra-consensus + "checkpoint::tests::block_higher_than_max_checkpoint_fail_test" + "checkpoint::tests::checkpoint_drop_cancel_test" + "checkpoint::tests::continuous_blockchain_no_restart" + "checkpoint::tests::continuous_blockchain_restart" + "checkpoint::tests::hard_coded_mainnet_test" + "checkpoint::tests::multi_item_checkpoint_list_test" + "checkpoint::tests::single_item_checkpoint_list_test" + "checkpoint::tests::wrong_checkpoint_hash_fail_test" + "router::tests::round_trip_checkpoint_test" + "router::tests::verify_checkpoint_test" + "router::tests::verify_fail_add_block_checkpoint_test" + "router::tests::verify_fail_no_coinbase_test" + ## zebra-grpc + "tests::snapshot::test_grpc_response_data" + "tests::vectors::test_grpc_methods_mocked" + ## zebra-rpc + "server::tests::vectors::rpc_server_spawn_parallel_threads" + "server::tests::vectors::rpc_server_spawn_single_thread" + "server::tests::vectors::rpc_server_spawn_unallocated_port_single_thread" + "server::tests::vectors::rpc_server_spawn_unallocated_port_single_thread_shutdown" + "server::tests::vectors::rpc_sever_spawn_unallocated_port_parallel_threads" + "server::tests::vectors::rpc_sever_spawn_unallocated_port_parallel_threads_shutdown" + ## zebrad + "components::inbound::tests::real_peer_set::inbound_block_empty_state_notfound" + "components::inbound::tests::real_peer_set::inbound_peers_empty_address_book" + "components::inbound::tests::real_peer_set::inbound_tx_empty_state_notfound" + "components::inbound::tests::real_peer_set::outbound_tx_partial_response_notfound" + "components::inbound::tests::real_peer_set::outbound_tx_unrelated_response_notfound" + ## zebrad – acceptance + "downgrade_state_format" + "new_state_format" + "regtest_block_templates_are_valid_block_submissions" + "start_args" + "start_no_args" + "update_state_format" + "zebra_state_conflict" + "zebra_zcash_listener_conflict" +] +++ lib.optionals stdenv.hostPlatform.isLinux [ + ## zebrad – acceptance + "non_blocking_logger" +] +## These tests seem to only fail when the `elasticsearch` feature is enabled on MacOS. +++ lib.optionals (stdenv.hostPlatform.isDarwin && lib.elem "elasticsearch" extraFeatures) [ + ## zebra-rpc + "methods::tests::snapshot::test_rpc_response_data" + "methods::tests::snapshot::test_z_get_treestate" + "methods::tests::vectors::rpc_getaddresstxids_invalid_arguments" + "methods::tests::vectors::rpc_getaddresstxids_response" + "methods::tests::vectors::rpc_getaddressutxos_response" + "methods::tests::vectors::rpc_getbestblockhash" + "methods::tests::vectors::rpc_getblock" + "methods::tests::vectors::rpc_getblockcount" + "methods::tests::vectors::rpc_getblockcount_empty_state" + "methods::tests::vectors::rpc_getblockhash" + "methods::tests::vectors::rpc_getmininginfo" + "methods::tests::vectors::rpc_getnetworksolps" + "methods::tests::vectors::rpc_getpeerinfo" + "methods::tests::vectors::rpc_getrawtransaction" + "methods::tests::vectors::rpc_submitblock_errors" + ## zebra-scan + "service::tests::scan_service_registers_keys_correctly" + "tests::vectors::scanning_zecpages_from_populated_zebra_state" + ## zebra-state + "service::read::tests::vectors::empty_read_state_still_responds_to_requests" + "service::read::tests::vectors::populated_read_state_responds_correctly" + "service::tests::chain_tip_sender_is_updated" + "service::tests::empty_state_still_responds_to_requests" + "service::tests::state_behaves_when_blocks_are_committed_in_order" + "service::tests::state_behaves_when_blocks_are_committed_out_of_order" + "service::tests::value_pool_is_updated" + ## zebra-state – basic + "check_transcripts_mainnet" + "check_transcripts_testnet" + ## zebrad + "components::inbound::tests::fake_peer_set::caches_getaddr_response" + "components::inbound::tests::fake_peer_set::inbound_block_height_lookahead_limit" + "components::inbound::tests::fake_peer_set::mempool_advertise_transaction_ids" + "components::inbound::tests::fake_peer_set::mempool_push_transaction" + "components::inbound::tests::fake_peer_set::mempool_requests_for_transactions" + "components::inbound::tests::fake_peer_set::mempool_transaction_expiration" + "components::mempool::tests::vector::mempool_cancel_downloads_after_network_upgrade" + "components::mempool::tests::vector::mempool_cancel_mined" + "components::mempool::tests::vector::mempool_failed_download_is_not_rejected" + "components::mempool::tests::vector::mempool_failed_verification_is_rejected" + "components::mempool::tests::vector::mempool_queue" + "components::mempool::tests::vector::mempool_reverifies_after_tip_change" + "components::mempool::tests::vector::mempool_service_basic" + "components::mempool::tests::vector::mempool_service_disabled" + ## zebrad – acceptance + "db_init_outside_future_executor" + "nu6_funding_streams_and_coinbase_balance" + "validate_regtest_genesis_block" +] diff --git a/nix/rust-toolchain.nix b/nix/rust-toolchain.nix new file mode 100644 index 00000000000..485668c3826 --- /dev/null +++ b/nix/rust-toolchain.nix @@ -0,0 +1,27 @@ +{ + fenix, + pkgs, +}: +## TODO: This conditional is a workaround for nix-community/fenix#178. Use the `else` +## unconditionally once that is fixed. +if pkgs.stdenv.isDarwin +then let + fnx = fenix.packages.${pkgs.system}; +in + fnx.combine [ + (fnx.stable.cargo.overrideAttrs (old: { + postBuild = '' + install_name_tool \ + -change "/usr/lib/libcurl.4.dylib" "${pkgs.curl.out}/lib/libcurl.4.dylib" \ + ./cargo/bin/cargo + ''; + })) + fnx.stable.clippy + fnx.stable.rustc + fnx.stable.rustfmt + ] +else + fenix.packages.${pkgs.system}.fromToolchainFile { + file = ../rust-toolchain.toml; + sha256 = "s1RPtyvDGJaX/BisLT+ifVfuhDT1nZkZ1NcK8sbwELM="; + } diff --git a/zebra-state/src/service/finalized_state/disk_db.rs b/zebra-state/src/service/finalized_state/disk_db.rs index 69be5a4585f..30a3f17b1d8 100644 --- a/zebra-state/src/service/finalized_state/disk_db.rs +++ b/zebra-state/src/service/finalized_state/disk_db.rs @@ -990,7 +990,7 @@ impl DiskDb { // // We don't attempt to guard against malicious symlinks created by attackers // (TOCTOU attacks). Zebra should not be run with elevated privileges. - if !old_path.starts_with(&cache_path) { + if !old_path.starts_with(cache_path) { info!("skipped reusing previous state cache: state is outside cache directory"); return; } diff --git a/zebrad/systemd/zebrad.service b/zebrad/systemd/zebrad.service index 92f9091fa62..965c4df6c36 100644 --- a/zebrad/systemd/zebrad.service +++ b/zebrad/systemd/zebrad.service @@ -6,7 +6,7 @@ # sudo cp target/release/zebrad /usr/bin # 3- Replace AssertPathExists and ExecStart with the location of your binary if needed. # 4- Place this file in systemd system folder, with the other systemd files: -# cp zebrad/systemd/zebrad.service /lib/systemd/system/ +# cp zebrad/systemd/zebrad.service /lib/systemd/system/ # 5- Start zebrad from systemd for the first time: # systemctl start zebrad.service # 6- Check status: @@ -26,5 +26,4 @@ StandardOutput=journal StandardError=journal [Install] -Alias=zebrad WantedBy=default.target