diff --git a/.version b/.version index ad2261920c0..5c18f9195b5 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.24.0 +2.25.0 diff --git a/doc/manual/rl-next/drop-vendored-toml11.md b/doc/manual/rl-next/drop-vendored-toml11.md deleted file mode 100644 index d1feeb7031a..00000000000 --- a/doc/manual/rl-next/drop-vendored-toml11.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -synopsis: Stop vendoring toml11 ---- - -We don't apply any patches to it, and vendoring it locks users into -bugs (it hasn't been updated since its introduction in late 2021). diff --git a/doc/manual/rl-next/harden-user-sandboxing.md b/doc/manual/rl-next/harden-user-sandboxing.md deleted file mode 100644 index a647acf2540..00000000000 --- a/doc/manual/rl-next/harden-user-sandboxing.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: Harden the user sandboxing -significance: significant -issues: ---- - -The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. diff --git a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md b/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md deleted file mode 100644 index 99be4148bf1..00000000000 --- a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -synopsis: "`nix-shell ` looks for `shell.nix`" -significance: significant -issues: -- 496 -- 2279 -- 4529 -- 5431 -- 11053 -prs: -- 11057 ---- - -`nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. - -Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. -This adjustment addresses a commonly reported problem. - -This also applies to `nix-shell` shebang scripts. Consider the following example: - -```shell -#!/usr/bin/env nix-shell -#!nix-shell -i bash -``` - -This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. - -The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. diff --git a/doc/manual/rl-next/pipe-operators.md b/doc/manual/rl-next/pipe-operators.md deleted file mode 100644 index b4cbe30e354..00000000000 --- a/doc/manual/rl-next/pipe-operators.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -synopsis: "Add `pipe-operators` experimental feature" -prs: -- 11131 ---- - -This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). - -The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. -*a* `|>` *b* is equivalent to the function application *b* *a*, and -*a* `<|` *b* is equivalent to the function application *a* *b*. - -For example: - -``` -nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 -9 - -nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 -7 -``` - -`<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. -These properties may change in future releases. - -See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. - -[pipe operators]: @docroot@/language/operators.md#pipe-operators diff --git a/doc/manual/rl-next/repl-doc-renders-doc-comments.md b/doc/manual/rl-next/repl-doc-renders-doc-comments.md deleted file mode 100644 index 05023697c96..00000000000 --- a/doc/manual/rl-next/repl-doc-renders-doc-comments.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -synopsis: "`nix-repl`'s `:doc` shows documentation comments" -significance: significant -issues: -- 3904 -- 10771 -prs: -- 1652 -- 9054 -- 11072 ---- - -`nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. -This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. - -Example: - -``` -nix-repl> :doc lib.toFunction -Function toFunction - … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 - - Turns any non-callable values into constant functions. Returns - callable values as is. - -Inputs - - v - - : Any value - -Examples - - :::{.example} - -## lib.trivial.toFunction usage example - - | nix-repl> lib.toFunction 1 2 - | 1 - | - | nix-repl> lib.toFunction (x: x + 1) 2 - | 3 - - ::: -``` - -Known limitations: -- It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. -- Some extensions to markdown are not yet supported, as you can see in the example above. - -We'd like to acknowledge Yingchi Long for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as @sternenseemann and Johannes Kirschbauer for their contributions, proposals, and their work on [RFC 145]. - -[RFC 145]: https://github.com/NixOS/rfcs/pull/145 diff --git a/doc/manual/rl-next/shebang-relative.md b/doc/manual/rl-next/shebang-relative.md deleted file mode 100644 index c887a598a05..00000000000 --- a/doc/manual/rl-next/shebang-relative.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -synopsis: "`nix-shell` shebang uses relative path" -prs: -- 5088 -- 11058 -issues: -- 4232 ---- - - -Relative [path](@docroot@/language/values.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary?highlight=base%20directory#gloss-base-directory). -Previously they were resolved relative to the current working directory. - -For example, consider the following script in `~/myproject/say-hi`: - -```shell -#!/usr/bin/env nix-shell -#!nix-shell --expr 'import ./shell.nix' -#!nix-shell --arg toolset './greeting-tools.nix' -#!nix-shell -i bash -hello -``` - -Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory; home in this example: - -```console -[hostname:~]$ ./myproject/say-hi -error: - … while calling the 'import' builtin - at «string»:1:2: - 1| (import ./shell.nix) - | ^ - - error: path '/home/user/shell.nix' does not exist -``` - -Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. - -```console -$ ./myproject/say-hi -Hello, world! -``` - -**Opt-out** - -This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. -The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. -This option will be removed in a future release. - -**`nix` command shebang** - -The experimental [`nix` command shebang](@docroot@/command-ref/new-cli/nix.md?highlight=shebang#shebang-interpreter) already behaves in this script-relative manner. - -Example: - -```shell -#!/usr/bin/env nix -#!nix develop -#!nix --expr ``import ./shell.nix`` -#!nix -c bash -hello -``` diff --git a/doc/manual/rl-next/tarball-fixes.md b/doc/manual/rl-next/tarball-fixes.md deleted file mode 100644 index c938e9db6c2..00000000000 --- a/doc/manual/rl-next/tarball-fixes.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: "Improve handling of tarballs that don't consist of a single top-level directory" -prs: -- 11195 ---- - -In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). - -Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 3918faeb2dd..7661f5f6287 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -28,6 +28,7 @@ - [Data Types](language/types.md) - [String context](language/string-context.md) - [Syntax and semantics](language/syntax.md) + - [Variables](language/variables.md) - [Identifiers](language/identifiers.md) - [Scoping rules](language/scope.md) - [String interpolation](language/string-interpolation.md) @@ -127,6 +128,7 @@ - [Contributing](development/contributing.md) - [Releases](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.24 (2024-07-31)](release-notes/rl-2.24.md) - [Release 2.23 (2024-06-03)](release-notes/rl-2.23.md) - [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md) - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md) diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index d3f5f9c1443..0b501788293 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -9,22 +9,26 @@ Most Nix commands interpret the following environment variables: - [`NIX_PATH`](#env-NIX_PATH) - A colon-separated list of directories used to look up the location of Nix - expressions using [paths](@docroot@/language/types.md#type-path) - enclosed in angle brackets (i.e., ``), - e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + A colon-separated list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). - If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: + This environment variable overrides the value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path). - 1. `$HOME/.nix-defexpr/channels` - 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` - 3. `/nix/var/nix/profiles/per-user/root/channels` + It can be extended using the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + + > **Example** + > + > ```bash + > $ export NIX_PATH=`/home/eelco/Dev:nixos-config=/etc/nixos + > ``` If `NIX_PATH` is set to an empty string, resolving search paths will always fail. - For example, attempting to use `` will produce: - error: file 'nixpkgs' was not found in the Nix search path + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval '' + > error: file 'nixpkgs' was not found in the Nix search path (add it using $NIX_PATH or -I) + > ``` - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index ddec30f5b3e..69a711bd5e6 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -297,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index a42909e2d13..69a7002072d 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -37,7 +37,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug” - + Print debug information. - `5` “Vomit” @@ -187,11 +187,12 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value* diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md index c9e981da626..861ee3e2092 100644 --- a/doc/manual/src/language/identifiers.md +++ b/doc/manual/src/language/identifiers.md @@ -16,13 +16,14 @@ An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character seq # Names -A name can be an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). +A *name* can be written as an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). > **Syntax** > > *name* → *identifier* | *string* -Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting attributes). +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. # Keywords diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md index 5c6aed38d31..9373324e2ff 100644 --- a/doc/manual/src/language/scope.md +++ b/doc/manual/src/language/scope.md @@ -1,14 +1,28 @@ # Scoping rules -Nix is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope), but with multiple scopes and shadowing rules. +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. -* primary scope: explicitly-bound variables - * [`let`](./syntax.md#let-expressions) - * [`inherit`](./syntax.md#inheriting-attributes) - * [function](./syntax.md#functions) arguments +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) -* secondary scope: implicitly-bound variables - * [`with`](./syntax.md#with-expressions) +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). -Primary scope takes precedence over secondary scope. -See [`with`](./syntax.md#with-expressions) for a detailed example. +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 00000000000..af6aff8a2c5 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 00000000000..5bcc1d79ced --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,324 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + +- The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stop vendoring `toml11` + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"nix32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein” Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/flake.nix b/flake.nix index 0bd08b70f31..f8cb2c3cc2d 100644 --- a/flake.nix +++ b/flake.nix @@ -341,6 +341,7 @@ ++ [ pkgs.buildPackages.cmake pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 00000000000..cddc1a6e70a --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,52 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 00000000000..abf9ed05b97 --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,45 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" +} \ No newline at end of file diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 00000000000..7a5c87d7dfb --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,184 @@ +#!/usr/bin/env nix +# vim: set filetype=python: +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +def is_bot(handle): + return handle.endswith("[bot]") + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 0fca5abf27e..c0c4ee734de 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,4 +1,5 @@ #!/usr/bin/env nix +# vim: set filetype=bash: #!nix shell .#changelog-d --command bash # --- CONFIGURATION --- @@ -151,6 +152,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea94d..7a2b3c0a79b 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4c4e2bd6f5e..73198856858 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -42,7 +42,7 @@ sub fetch { my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -91,7 +91,7 @@ sub getStorePath { sub copyManual { my $manual; eval { - $manual = getStorePath("build.x86_64-linux", "doc"); + $manual = getStorePath("build.nix.x86_64-linux", "doc"); }; if ($@) { warn "$@"; @@ -240,12 +240,12 @@ sub downloadFile { # Upload nix-fallback-paths.nix. write_file("$tmpDir/fallback-paths.nix", "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " riscv64-linux = \"" . getStorePath("buildCross.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . "}\n"); # Upload release files to S3. diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf73..22c114a2077 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index decadd751cc..fcef92487cb 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -91,75 +91,11 @@ MixEvalArgs::MixEvalArgs() .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/types.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index adf8b65b1a3..1764b49f321 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -14,6 +14,16 @@ #include "nix_api_util.h" #include +#ifndef __has_c_attribute +# define __has_c_attribute(x) 0 +#endif + +#if __has_c_attribute(deprecated) +# define NIX_DEPRECATED(msg) [[deprecated(msg)]] +#else +# define NIX_DEPRECATED(msg) +#endif + #ifdef __cplusplus extern "C" { #endif @@ -45,7 +55,7 @@ typedef struct EvalState EvalState; // nix::EvalState * @see nix_value_incref, nix_value_decref */ typedef struct nix_value nix_value; -[[deprecated("use nix_value instead")]] typedef nix_value Value; +NIX_DEPRECATED("use nix_value instead") typedef nix_value Value; // Function prototypes /** diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 30a8c5c588a..0cfc14c1b74 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -79,19 +79,24 @@ struct EvalSettings : Config This setting determines the value of [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile). - The default value is + - The configuration setting is overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) + environment variable. + - `NIX_PATH` is overridden by [specifying the setting as the command line flag](@docroot@/command-ref/conf-file.md#command-line-flags) `--nix-path`. + - Any current value is extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`. - ``` - $HOME/.nix-defexpr/channels - nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs - $NIX_STATE_DIR/profiles/per-user/root/channels - ``` + If the respective paths are accessible, the default values are: - It can be overridden with the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH) or the [`-I` command line option](@docroot@/command-ref/opt-common.md#opt-I). + - `$HOME/.nix-defexpr/channels` + - `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs` + - `$NIX_STATE_DIR/profiles/per-user/root/channels` + + See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details. > **Note** > - > If [pure evaluation](#conf-pure-eval) is enabled, `nixPath` evaluates to the empty list `[ ]`. + > If [restricted evaluation](@docroot@/command-ref/conf-file.md#conf-restrict-eval) is enabled, the default value is empty. + > + > If [pure evaluation](#conf-pure-eval) is enabled, `builtins.nixPath` *always* evaluates to the empty list `[ ]`. )", {}, false}; Setting currentSystem{ diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 4bb5c92046b..8ad0d9ad714 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -20,6 +20,7 @@ struct StringToken operator std::string_view() const { return {p, l}; } }; +// This type must be trivially copyable; see YYLTYPE_IS_TRIVIAL in parser.y. struct ParserLocation { int beginOffset; @@ -86,7 +87,7 @@ struct ParserState void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); - void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos); + void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc); Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); Expr * stripIndentation(const PosIdx pos, std::vector>> && es); @@ -110,11 +111,12 @@ inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx pre }); } -inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos) +inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc) { AttrPath::iterator i; // All attrpaths have at least one attr assert(!attrPath.empty()); + auto pos = at(loc); // Checking attrPath validity. // =========================== for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { @@ -179,6 +181,12 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * } else { attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); } + + auto it = lexerState.positionToDocComment.find(pos); + if (it != lexerState.positionToDocComment.end()) { + e->setDocComment(it->second); + lexerState.positionToDocComment.emplace(at(exprLoc), it->second); + } } inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 9ad41c148dd..f2ccca7fcdd 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,4 +1,4 @@ -%glr-parser +%define api.location.type { ::nix::ParserLocation } %define api.pure %locations %define parse.error verbose @@ -8,8 +8,7 @@ %parse-param { nix::ParserState * state } %lex-param { void * scanner } %lex-param { nix::ParserState * state } -%expect 1 -%expect-rr 1 +%expect 0 %code requires { @@ -27,7 +26,17 @@ #include "eval-settings.hh" #include "parser-state.hh" -#define YYLTYPE ::nix::ParserLocation +// Bison seems to have difficulty growing the parser stack when using C++ with +// a custom location type. This undocumented macro tells Bison that our +// location type is "trivially copyable" in C++-ese, so it is safe to use the +// same memcpy macro it uses to grow the stack that it uses with its own +// default location type. Without this, we get "error: memory exhausted" when +// parsing some large Nix files. Our other options are to increase the initial +// stack size (200 by default) to be as large as we ever want to support (so +// that growing the stack is unnecessary), or redefine the stack-relocation +// macro ourselves (which is also undocumented). +#define YYLTYPE_IS_TRIVIAL 1 + #define YY_DECL int yylex \ (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state) @@ -77,7 +86,7 @@ YY_DECL; using namespace nix; -#define CUR_POS state->at(*yylocp) +#define CUR_POS state->at(yyloc) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) @@ -133,8 +142,8 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) { %type expr_select expr_simple expr_app %type expr_pipe_from expr_pipe_into %type expr_list -%type binds -%type formals +%type binds binds1 +%type formals formal_set %type formal %type attrpath %type attrs @@ -180,22 +189,22 @@ expr_function $$ = me; SET_DOC_POS(me, @1); } - | '{' formals '}' ':' expr_function - { auto me = new ExprLambda(CUR_POS, state->validateFormals($2), $5); + | formal_set ':' expr_function[body] + { auto me = new ExprLambda(CUR_POS, state->validateFormals($formal_set), $body); $$ = me; SET_DOC_POS(me, @1); } - | '{' formals '}' '@' ID ':' expr_function + | formal_set '@' ID ':' expr_function[body] { - auto arg = state->symbols.create($5); - auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7); + auto arg = state->symbols.create($ID); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body); $$ = me; SET_DOC_POS(me, @1); } - | ID '@' '{' formals '}' ':' expr_function + | ID '@' formal_set ':' expr_function[body] { - auto arg = state->symbols.create($1); - auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7); + auto arg = state->symbols.create($ID); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body); $$ = me; SET_DOC_POS(me, @1); } @@ -311,11 +320,13 @@ expr_simple /* Let expressions `let {..., body = ...}' are just desugared into `(rec {..., body = ...}).body'. */ | LET '{' binds '}' - { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->s.body); } + { $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(noPos, $3, state->s.body); } | REC '{' binds '}' - { $3->recursive = true; $$ = $3; } - | '{' binds '}' - { $$ = $2; } + { $3->recursive = true; $3->pos = CUR_POS; $$ = $3; } + | '{' binds1 '}' + { $2->pos = CUR_POS; $$ = $2; } + | '{' '}' + { $$ = new ExprAttrs(CUR_POS); } | '[' expr_list ']' { $$ = $2; } ; @@ -364,52 +375,50 @@ ind_string_parts ; binds - : binds attrpath '=' expr ';' { - $$ = $1; - - auto pos = state->at(@2); - auto exprPos = state->at(@4); - { - auto it = state->lexerState.positionToDocComment.find(pos); - if (it != state->lexerState.positionToDocComment.end()) { - $4->setDocComment(it->second); - state->lexerState.positionToDocComment.emplace(exprPos, it->second); - } - } + : binds1 + | { $$ = new ExprAttrs; } + ; - state->addAttr($$, std::move(*$2), $4, pos); - delete $2; +binds1 + : binds1[accum] attrpath '=' expr ';' + { $$ = $accum; + state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); + delete $attrpath; } - | binds INHERIT attrs ';' - { $$ = $1; - for (auto & [i, iPos] : *$3) { - if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos); - $$->attrs.emplace( + | binds[accum] INHERIT attrs ';' + { $$ = $accum; + for (auto & [i, iPos] : *$attrs) { + if ($accum->attrs.find(i.symbol) != $accum->attrs.end()) + state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos); + $accum->attrs.emplace( i.symbol, ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited)); } - delete $3; + delete $attrs; } - | binds INHERIT '(' expr ')' attrs ';' - { $$ = $1; - if (!$$->inheritFromExprs) - $$->inheritFromExprs = std::make_unique>(); - $$->inheritFromExprs->push_back($4); - auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1); - for (auto & [i, iPos] : *$6) { - if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos); - $$->attrs.emplace( + | binds[accum] INHERIT '(' expr ')' attrs ';' + { $$ = $accum; + if (!$accum->inheritFromExprs) + $accum->inheritFromExprs = std::make_unique>(); + $accum->inheritFromExprs->push_back($expr); + auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1); + for (auto & [i, iPos] : *$attrs) { + if ($accum->attrs.find(i.symbol) != $accum->attrs.end()) + state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos); + $accum->attrs.emplace( i.symbol, ExprAttrs::AttrDef( new ExprSelect(iPos, from, i.symbol), iPos, ExprAttrs::AttrDef::Kind::InheritedFrom)); } - delete $6; + delete $attrs; + } + | attrpath '=' expr ';' + { $$ = new ExprAttrs; + state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); + delete $attrpath; } - | { $$ = new ExprAttrs(state->at(@0)); } ; attrs @@ -467,15 +476,19 @@ expr_list | { $$ = new ExprList; } ; +formal_set + : '{' formals ',' ELLIPSIS '}' { $$ = $formals; $$->ellipsis = true; } + | '{' ELLIPSIS '}' { $$ = new Formals; $$->ellipsis = true; } + | '{' formals ',' '}' { $$ = $formals; $$->ellipsis = false; } + | '{' formals '}' { $$ = $formals; $$->ellipsis = false; } + | '{' '}' { $$ = new Formals; $$->ellipsis = false; } + ; + formals - : formal ',' formals - { $$ = $3; $$->formals.emplace_back(*$1); delete $1; } + : formals[accum] ',' formal + { $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; } | formal - { $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; } - | - { $$ = new Formals; $$->ellipsis = false; } - | ELLIPSIS - { $$ = new Formals; $$->ellipsis = true; } + { $$ = new Formals; $$->formals.emplace_back(*$formal); delete $formal; } ; formal diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0b3b19b5764..7ceb84f0e39 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1843,45 +1843,130 @@ static RegisterPrimOp primop_findFile(PrimOp { .doc = R"( Find *lookup-path* in *search-path*. - A search path is represented list of [attribute sets](./types.md#attribute-set) with two attributes: + [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](#builtins-nixPath): + + ```nix + + ``` + + is equivalent to: + + ```nix + builtins.findFile builtins.nixPath "nixpkgs" + ``` + + A search path is represented as a list of [attribute sets](./types.md#attribute-set) with two attributes: - `prefix` is a relative path. - `path` denotes a file system location - The exact syntax depends on the command line interface. Examples of search path attribute sets: + - ``` + { + prefix = ""; + path = "/nix/var/nix/profiles/per-user/root/channels"; + } + ``` - ``` { prefix = "nixos-config"; path = "/etc/nixos/configuration.nix"; } ``` - - ``` { - prefix = ""; - path = "/nix/var/nix/profiles/per-user/root/channels"; + prefix = "nixpkgs"; + path = "https://github.com/NixOS/nixpkgs/tarballs/master"; + } + ``` + - ``` + { + prefix = "nixpkgs"; + path = "channel:nixpkgs-unstable"; + } + ``` + - ``` + { + prefix = "flake-compat"; + path = "flake:github:edolstra/flake-compat"; } ``` The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/types.md#type-path) of the match: - - If *lookup-path* matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. - Note that the `path` may need to be downloaded at this point to look inside. + - If a prefix of `lookup-path` matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. + The contents of `path` may need to be downloaded at this point to look inside. + - If the suffix is found inside that directory, then the entry is a match. The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. - [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](#builtins-nixPath): + > **Example** + > + > A *search-path* value + > + > ``` + > [ + > { + > prefix = ""; + > path = "/home/eelco/Dev"; + > } + > { + > prefix = "nixos-config"; + > path = "/etc/nixos"; + > } + > ] + > ``` + > + > and a *lookup-path* value `"nixos-config"` will cause Nix to try `/home/eelco/Dev/nixos-config` and `/etc/nixos` in that order and return the first path that exists. - ```nix - - ``` + If `path` starts with `http://` or `https://`, it is interpreted as the URL of a tarball that will be downloaded and unpacked to a temporary location. + The tarball must consist of a single top-level directory. - is equivalent to: + The URLs of the tarballs from the official `nixos.org` channels can be abbreviated as `channel:`. + See [documentation on `nix-channel`](@docroot@/command-ref/nix-channel.md) for details about channels. - ```nix - builtins.findFile builtins.nixPath "nixpkgs" - ``` + > **Example** + > + > These two search path entries are equivalent: + > + > - ``` + > { + > prefix = "nixpkgs"; + > path = "channel:nixpkgs-unstable"; + > } + > ``` + > - ``` + > { + > prefix = "nixpkgs"; + > path = "https://nixos.org/channels/nixos-unstable/nixexprs.tar.xz"; + > } + > ``` + + Search paths can also point to source trees using [flake URLs](@docroot@/command-ref/new-cli/nix3-flake.md#url-like-syntax). + + + > **Example** + > + > The search path entry + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:nixpkgs"; + > } + > ``` + > specifies that the prefix `nixpkgs` shall refer to the source tree downloaded from the `nixpkgs` entry in the flake registry. + > + > Similarly + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:github:nixos/nixpkgs/nixos-22.05"; + > } + > ``` + > + > makes `` refer to a particular branch of the `NixOS/nixpkgs` repository on GitHub. )", .fun = prim_findFile, }); @@ -4731,6 +4816,13 @@ void EvalState::createBaseEnv() .doc = R"( The value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path): a list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval --expr "builtins.nixPath" -I foo=bar --no-pure-eval + > [ { path = "bar"; prefix = "foo"; } ] + > ``` + Lookup path expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.findFile`](./builtins.html#builtins-findFile): diff --git a/src/libflake/flake/nix-flake.pc.in b/src/libflake/flake/nix-flake.pc.in new file mode 100644 index 00000000000..10c52f5e9bb --- /dev/null +++ b/src/libflake/flake/nix-flake.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Requires: nix-util nix-store nix-expr +Libs: -L${libdir} -lnixflake +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libflake/local.mk b/src/libflake/local.mk index 2cceda2bf25..5e604ef3abf 100644 --- a/src/libflake/local.mk +++ b/src/libflake/local.mk @@ -15,3 +15,8 @@ libflake_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetcher libflake_LDFLAGS += $(THREAD_LDFLAGS) libflake_LIBS = libutil libstore libfetchers libexpr + +$(eval $(call install-file-in, $(buildprefix)$(d)/flake/nix-flake.pc, $(libdir)/pkgconfig, 0644)) + +$(foreach i, $(wildcard src/libflake/flake/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 7df16d64a24..ef3326cd64d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1025,19 +1025,20 @@ void processConnection( #endif /* Exchange the greeting. */ - WorkerProto::Version clientVersion = + auto [protoVersion, features] = WorkerProto::BasicServerConnection::handshake( - to, from, PROTOCOL_VERSION); + to, from, PROTOCOL_VERSION, WorkerProto::allFeatures); - if (clientVersion < 0x10a) + if (protoVersion < 0x10a) throw Error("the Nix client version is too old"); WorkerProto::BasicServerConnection conn; conn.to = std::move(to); conn.from = std::move(from); - conn.protoVersion = clientVersion; + conn.protoVersion = protoVersion; + conn.features = features; - auto tunnelLogger = new TunnelLogger(conn.to, clientVersion); + auto tunnelLogger = new TunnelLogger(conn.to, protoVersion); auto prevLogger = nix::logger; // FIXME if (!recursive) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ebb0864c555..555936c186f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -73,8 +73,11 @@ void RemoteStore::initConnection(Connection & conn) StringSink saved; TeeSource tee(conn.from, saved); try { - conn.protoVersion = WorkerProto::BasicClientConnection::handshake( - conn.to, tee, PROTOCOL_VERSION); + auto [protoVersion, features] = WorkerProto::BasicClientConnection::handshake( + conn.to, tee, PROTOCOL_VERSION, + WorkerProto::allFeatures); + conn.protoVersion = protoVersion; + conn.features = features; } catch (SerialisationError & e) { /* In case the other side is waiting for our input, close it. */ @@ -88,6 +91,9 @@ void RemoteStore::initConnection(Connection & conn) static_cast(conn) = conn.postHandshake(*this); + for (auto & feature : conn.features) + debug("negotiated feature '%s'", feature); + auto ex = conn.processStderrReturn(); if (ex) std::rethrow_exception(ex); } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 92ab47cd66d..21175b1ebfd 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -220,8 +220,6 @@ std::string S3BinaryCacheStoreConfig::doc() struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { - std::string bucketName; - Stats stats; S3Helper s3Helper; diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index 93d13d48e55..a47dbb689d8 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -5,6 +5,8 @@ namespace nix { +const std::set WorkerProto::allFeatures{}; + WorkerProto::BasicClientConnection::~BasicClientConnection() { try { @@ -137,8 +139,21 @@ void WorkerProto::BasicClientConnection::processStderr(bool * daemonException, S } } -WorkerProto::Version -WorkerProto::BasicClientConnection::handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion) +static std::set +intersectFeatures(const std::set & a, const std::set & b) +{ + std::set res; + for (auto & x : a) + if (b.contains(x)) + res.insert(x); + return res; +} + +std::tuple> WorkerProto::BasicClientConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) { to << WORKER_MAGIC_1 << localVersion; to.flush(); @@ -153,11 +168,24 @@ WorkerProto::BasicClientConnection::handshake(BufferedSink & to, Source & from, if (GET_PROTOCOL_MINOR(daemonVersion) < 10) throw Error("the Nix daemon version is too old"); - return std::min(daemonVersion, localVersion); + auto protoVersion = std::min(daemonVersion, localVersion); + + /* Exchange features. */ + std::set daemonFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + to << supportedFeatures; + to.flush(); + daemonFeatures = readStrings>(from); + } + + return {protoVersion, intersectFeatures(daemonFeatures, supportedFeatures)}; } -WorkerProto::Version -WorkerProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion) +std::tuple> WorkerProto::BasicServerConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) { unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_1) @@ -165,7 +193,18 @@ WorkerProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, to << WORKER_MAGIC_2 << localVersion; to.flush(); auto clientVersion = readInt(from); - return std::min(clientVersion, localVersion); + + auto protoVersion = std::min(clientVersion, localVersion); + + /* Exchange features. */ + std::set clientFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + clientFeatures = readStrings>(from); + to << supportedFeatures; + to.flush(); + } + + return {protoVersion, intersectFeatures(clientFeatures, supportedFeatures)}; } WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandshake(const StoreDirConfig & store) diff --git a/src/libstore/worker-protocol-connection.hh b/src/libstore/worker-protocol-connection.hh index 38287d08e31..9c96195b5f4 100644 --- a/src/libstore/worker-protocol-connection.hh +++ b/src/libstore/worker-protocol-connection.hh @@ -23,6 +23,11 @@ struct WorkerProto::BasicConnection */ WorkerProto::Version protoVersion; + /** + * The set of features that both sides support. + */ + std::set features; + /** * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the * factored out serve protocol serializers with a @@ -72,8 +77,8 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection /** * Establishes connection, negotiating version. * - * @return the version provided by the other side of the - * connection. + * @return the minimum version supported by both sides and the set + * of protocol features supported by both sides. * * @param to Taken by reference to allow for various error handling * mechanisms. @@ -82,8 +87,15 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection * handling mechanisms. * * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support */ - static Version handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion); + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); /** * After calling handshake, must call this to exchange some basic @@ -138,8 +150,15 @@ struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection * handling mechanisms. * * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support */ - static WorkerProto::Version handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion); + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); /** * After calling handshake, must call this to exchange some basic diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 9fc16d0153e..c356fa1bf37 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -11,7 +11,9 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 37) +/* Note: you generally shouldn't change the protocol version. Define a + new `WorkerProto::Feature` instead. */ +#define PROTOCOL_VERSION (1 << 8 | 38) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -131,6 +133,10 @@ struct WorkerProto { WorkerProto::Serialise::write(store, conn, t); } + + using Feature = std::string; + + static const std::set allFeatures; }; enum struct WorkerProto::Op : uint64_t diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index c2b1ac500f4..0bc46d7464c 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -137,7 +137,7 @@ std::optional getSelfExe() { static auto cached = []() -> std::optional { - #if __linux__ + #if __linux__ || __GNU__ return readLink("/proc/self/exe"); #elif __APPLE__ char buf[1024]; diff --git a/src/nix/meson.build b/src/nix/meson.build index cf535b7513b..54cf90410fc 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -22,6 +22,7 @@ deps_private_maybe_subproject = [ dependency('nix-util'), nix_store, dependency('nix-expr'), + dependency('nix-flake'), dependency('nix-fetchers'), dependency('nix-main'), dependency('nix-cmd'), diff --git a/src/nix/nix.md b/src/nix/nix.md index f958ce09acc..56587d0b233 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -59,9 +59,13 @@ These are command line arguments that represent something that can be realised i The following types of installable are supported by most commands: - [Flake output attribute](#flake-output-attribute) (experimental) + - This is the default - [Store path](#store-path) + - This is assumed if the argument is a Nix store path or a symlink to a Nix store path - [Nix file](#nix-file), optionally qualified by an attribute path + - Specified with `--file`/`-f` - [Nix expression](#nix-expression), optionally qualified by an attribute path + - Specified with `--expr`/`-E` For most commands, if no installable is specified, `.` is assumed. That is, Nix will operate on the default flake output attribute of the flake in the current directory. @@ -178,9 +182,10 @@ that contains programs, and a `dev` output that provides development artifacts like C/C++ header files. The outputs on which `nix` commands operate are determined as follows: -* You can explicitly specify the desired outputs using the syntax - *installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can - obtain the `dev` and `static` outputs of the `glibc` package: +* You can explicitly specify the desired outputs using the syntax *installable*`^`*output1*`,`*...*`,`*outputN* — that is, a caret followed immediately by a comma-separated list of derivation outputs to select. + For installables specified as [Flake output attributes](#flake-output-attribute) or [Store paths](#store-path), the output is specified in the same argument: + + For example, you can obtain the `dev` and `static` outputs of the `glibc` package: ```console # nix build 'nixpkgs#glibc^dev,static' @@ -195,6 +200,19 @@ operate are determined as follows: … ``` + For `-e`/`--expr` and `-f`/`--file`, the derivation output is specified as part of the attribute path: + + ```console + $ nix build -f '' 'glibc^dev,static' + $ nix build --impure -E 'import { }' 'glibc^dev,static' + ``` + + This syntax is the same even if the actual attribute path is empty: + + ```console + $ nix build -E 'let pkgs = import { }; in pkgs.glibc' '^dev,static' + ``` + * You can also specify that *all* outputs should be used using the syntax *installable*`^*`. For example, the following shows the size of all outputs of the `glibc` package in the binary cache: diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index 000b30823f8..d05fac4e7f8 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -184,7 +184,7 @@ expect() { shift "$@" && res=0 || res="$?" # also match "negative" codes, which wrap around to >127 - if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then + if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi @@ -199,7 +199,7 @@ expectStderr() { shift "$@" 2>&1 && res=0 || res="$?" # also match "negative" codes, which wrap around to >127 - if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then + if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi @@ -287,7 +287,7 @@ checkGrepArgs() { for arg in "$@"; do if [[ "$arg" != "${arg//$'\n'/_}" ]]; then echo "$(callerPrefix)newline not allowed in arguments; grep would try each line individually as if connected by an OR operator" >&2 - return -101 + return 155 # = -101 mod 256 fi done } diff --git a/tests/functional/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp index 393c454dd31..96e87b2aa91 100644 --- a/tests/functional/lang/parse-fail-undef-var-2.err.exp +++ b/tests/functional/lang/parse-fail-undef-var-2.err.exp @@ -1,4 +1,4 @@ -error: syntax error, unexpected ':', expecting '}' +error: syntax error, unexpected ':', expecting '}' or ',' at «stdin»:3:13: 2| 3| f = {x, y : ["baz" "bar" z "bat"]}: x + y; diff --git a/tests/functional/nix_path.sh b/tests/functional/nix_path.sh index 7e6a0458d10..90cba1f0c9c 100755 --- a/tests/functional/nix_path.sh +++ b/tests/functional/nix_path.sh @@ -22,13 +22,13 @@ nix-instantiate --eval -E '' --restrict-eval # # | precedence | hard-coded | nix-path in file | extra-nix-path in file | nix-path in env | extra-nix-path in env | NIX_PATH | nix-path | extra-nix-path | -I | # |------------------------|------------|------------------|------------------------|-----------------|-----------------------|-----------|-----------|-----------------|-----------------| -# | hard-coded | x | ^override | ^append | ^override | ^append | ^override | ^override | ^append | ^append | -# | nix-path in file | | last wins | ^append | ^override | ^append | ^override | ^override | ^append | ^append | -# | extra-nix-path in file | | | append in order | ^override | ^append | ^override | ^override | ^append | ^append | -# | nix-path in env | | | | last wins | ^append | ^override | ^override | ^append | ^append | -# | extra-nix-path in env | | | | | append in order | ^override | ^override | ^append | ^append | -# | NIX_PATH | | | | | | x | ^override | ^append | ^append | -# | nix-path | | | | | | | last wins | ^append | ^append | +# | hard-coded | x | ^override | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in file | | last wins | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in file | | | append in order | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in env | | | | last wins | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in env | | | | | append in order | ^override | ^override | ^append | ^prepend | +# | NIX_PATH | | | | | | x | ^override | ^append | ^prepend | +# | nix-path | | | | | | | last wins | ^append | ^prepend | # | extra-nix-path | | | | | | | | append in order | append in order | # | -I | | | | | | | | | append in order | @@ -59,6 +59,8 @@ echo "nix-path = test=$TEST_ROOT/from-nix-path-file" >> "$test_nix_conf" # -I extends NIX_PATH [[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# -I takes precedence over NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test) = $TEST_ROOT/from-I ]] # if -I does not have the desired entry, the value from NIX_PATH is used [[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index c1512001093..bbea9ed7513 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -658,15 +658,15 @@ TEST_F(WorkerProtoTest, handshake_log) FdSink out { toServer.writeSide.get() }; FdSource in0 { toClient.readSide.get() }; TeeSource in { in0, toClientLog }; - clientResult = WorkerProto::BasicClientConnection::handshake( - out, in, defaultVersion); + clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + out, in, defaultVersion, {})); }); { FdSink out { toClient.writeSide.get() }; FdSource in { toServer.readSide.get() }; WorkerProto::BasicServerConnection::handshake( - out, in, defaultVersion); + out, in, defaultVersion, {}); }; thread.join(); @@ -675,6 +675,33 @@ TEST_F(WorkerProtoTest, handshake_log) }); } +TEST_F(WorkerProtoTest, handshake_features) +{ + Pipe toClient, toServer; + toClient.create(); + toServer.create(); + + std::tuple> clientResult; + + auto clientThread = std::thread([&]() { + FdSink out { toServer.writeSide.get() }; + FdSource in { toClient.readSide.get() }; + clientResult = WorkerProto::BasicClientConnection::handshake( + out, in, 123, {"bar", "aap", "mies", "xyzzy"}); + }); + + FdSink out { toClient.writeSide.get() }; + FdSource in { toServer.readSide.get() }; + auto daemonResult = WorkerProto::BasicServerConnection::handshake( + out, in, 456, {"foo", "bar", "xyzzy"}); + + clientThread.join(); + + EXPECT_EQ(clientResult, daemonResult); + EXPECT_EQ(std::get<0>(clientResult), 123); + EXPECT_EQ(std::get<1>(clientResult), std::set({"bar", "xyzzy"})); +} + /// Has to be a `BufferedSink` for handshake. struct NullBufferedSink : BufferedSink { void writeUnbuffered(std::string_view data) override { } @@ -686,8 +713,8 @@ TEST_F(WorkerProtoTest, handshake_client_replay) NullBufferedSink nullSink; StringSource in { toClientLog }; - auto clientResult = WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion); + auto clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); EXPECT_EQ(clientResult, defaultVersion); }); @@ -705,13 +732,13 @@ TEST_F(WorkerProtoTest, handshake_client_truncated_replay_throws) if (len < 8) { EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), EndOfFile); } else { // Not sure why cannot keep on checking for `EndOfFile`. EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), Error); } } @@ -734,17 +761,17 @@ TEST_F(WorkerProtoTest, handshake_client_corrupted_throws) // magic bytes don't match EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), Error); } else if (idx < 8 || idx >= 12) { // Number out of bounds EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), SerialisationError); } else { - auto ver = WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion); + auto ver = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); // `std::min` of this and the other version saves us EXPECT_EQ(ver, defaultVersion); } diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 8a3ca8561e5..2b73d323b8d 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -17,6 +17,10 @@ # define FS_ROOT FS_SEP #endif +#ifndef PATH_MAX +# define PATH_MAX 4096 +#endif + namespace nix { /* ----------- tests for util.hh ------------------------------------------------*/