Enforce that every toplevel field of organist has some documentation

doc/filegen.md

@@ -29,7 +29,10 @@ This has two main advantages:
 
 ## Usage
 
-To re-generate the files run:
+The files specified in `files.*` are regenerated automatically when entering
+the devshell (unless `filegen_hook` is set to false).
+
+They can also be regenerated manually with:
 
 ```bash
 nix run .#regenerate-files

flake.nix

@@ -44,17 +44,15 @@
         then {}
         else {
           inherit nickelOutputs;
-          packages = nickelOutputs.packages or {} // nickelOutputs.flake.packages or {};
-          checks = nickelOutputs.flake.checks or {};
+          packages = nickelOutputs.packages or {};
+          checks = nickelOutputs.checks or {};
           # Can't define this app in Nickel, yet
           apps =
             {
               regenerate-lockfile = lib.regenerateLockFileApp lockFileContents;
             }
-            // nickelOutputs.flake.apps or {};
-          # We can't just copy `shells` to `flake.devShells` in the contract
-          # because of a bug in Nickel: https://github.com/tweag/nickel/issues/1630
-          devShells = nickelOutputs.shells or {} // nickelOutputs.flake.devShells or {};
+            // nickelOutputs.apps or {};
+          devShells = nickelOutputs.devShells or {};
         });
 
     computedOutputs = outputsFromNickel ./. (inputs // {organist = self;}) {

lib/files.ncl

@@ -11,7 +11,18 @@ let File = {
     | doc m%"
         The content of the file.
       "%
-    | nix.derivation.NixString,
+    | nix.derivation.NullOr nix.derivation.NixString
+    | default
+    = null,
+  file
+    | doc "File from which to read the body of the script"
+    | nix.derivation.NullOr nix.derivation.NixString
+    | default
+    =
+      if content == null then
+        null
+      else
+        nix.builtins.to_file "generated-content" content,
   materialisation_method
     : [| 'Symlink, 'Copy |]
     | doc m%"
@@ -34,7 +45,6 @@ let NormaliseTargets = fun label files =>
 in
 let regenerate_files | Files -> nix.derivation.Derivation = fun files_to_generate =>
     let regnerate_one | String -> File -> nix.derivation.NixString = fun key file_descr =>
-        let file_content = file_descr.content in
         let target = file_descr.target in
         let copy_command =
           match {
@@ -43,17 +53,17 @@ let regenerate_files | Files -> nix.derivation.Derivation = fun files_to_generat
           }
             file_descr.materialisation_method
         in
-        let file_in_store =
-          nix.builtins.to_file
-            (nix.utils.escape_drv_name key)
-            file_content
-        in
         nix-s%"
-            rm -f %{target}
-            echo "Regenerating %{target}"
-	    target_dir=$(dirname %{target})
-	    test "${target_dir}" != "." && mkdir -p "${target_dir}"
-            %{copy_command} %{file_in_store} %{target}
+            if [[ ! -f "%{target}" ]] || [[ $(cat "%{target}") != $(cat "%{file_descr.file}") ]]; then
+              rm -f %{target}
+              echo "Regenerating %{target}"
+              target_dir=$(dirname %{target})
+              test "${target_dir}" != "." && mkdir -p "${target_dir}"
+              # XXX: If `source.file` is set explicitely to a relative path
+              # and `materialisation_method` is `'Symlink`, this will link to the
+              # original file, not one in the store. Not sure that's what we want.
+              %{copy_command} %{file_descr.file} %{target}
+            fi
           "%
       in
     {
@@ -83,10 +93,31 @@ let regenerate_files | Files -> nix.derivation.Derivation = fun files_to_generat
         Set of files that should be generated in the project's directory.
       "%
       = {},
+    filegen_hook.enable
+      | Bool
+      | doc m%"
+        Enable a hook that will automatically regenerate the files managed by
+        Nickel when entering the shell.
+      "%
+      | default
+      = true,
     flake.apps, # Forward declaration. Not great but would need some refactor to fix
+    shells,
   },
   config | Schema = {
       files,
+      filegen_hook,
+      shells,
       flake.apps.regenerate-files.program = nix-s%"%{regenerate_files files}/bin/regenerate-files"%,
+
+      shells.build.hooks =
+        if filegen_hook.enable then
+          {
+            filegen_hook = nix-s%"
+            %{regenerate_files files}/bin/regenerate-files
+          "%
+          }
+        else
+          {},
     },
 }

lib/lib.nix

@@ -157,7 +157,7 @@
       in
       let organist = (import "${src}/nickel.lock.ncl").organist in
 
-      let nickel_expr | (organist.OrganistExpression & {..}) =
+      let nickel_expr =
         import "${src}/${nickelFile}" in
 
       nickel_expr & params
@@ -169,28 +169,24 @@
       passAsFile = ["expectedLockfileContents"];
     } (
       if needNewLockfile
-      then
-        lib.warn ''
-          Lockfile contents are outdated. Please run "nix run .#regenerate-lockfile" to update them.
-        ''
-        ''
-          cp -r "${sources}" sources
-          if [ -f sources/nickel.lock.ncl ]; then
-            chmod +w sources sources/nickel.lock.ncl
-          else
-            chmod +w sources
-          fi
-          cp $expectedLockfileContentsPath sources/nickel.lock.ncl
-          cat > eval.ncl <<EOF
-          ${nickelWithImports "sources"}
-          EOF
-          ${nickel}/bin/nickel export eval.ncl > $out
-        ''
+      then ''
+        cp -r "${sources}" sources
+        if [ -f sources/nickel.lock.ncl ]; then
+          chmod +w sources sources/nickel.lock.ncl
+        else
+          chmod +w sources
+        fi
+        cp $expectedLockfileContentsPath sources/nickel.lock.ncl
+        cat > eval.ncl <<EOF
+        ${nickelWithImports "sources"}
+        EOF
+        ${nickel}/bin/nickel export eval.ncl --field config.flake > $out
+      ''
       else ''
         cat > eval.ncl <<EOF
         ${nickelWithImports sources}
         EOF
-        ${nickel}/bin/nickel export eval.ncl > $out
+        ${nickel}/bin/nickel export eval.ncl --field config.flake > $out
       ''
     );
 
@@ -212,9 +208,14 @@
     nickelResult = callNickel {
       inherit nickelFile baseDir flakeInputs lockFileContents;
     };
+    enrichedFlakeInputs =
+      flakeInputs
+      // {
+        "%%organist_internal".nickelLock = builtins.toFile "nickel.lock.ncl" (buildLockFileContents lockFileContents);
+      };
   in
     {rawNickel = nickelResult;}
-    // (importFromNickel flakeInputs system baseDir (builtins.fromJSON
+    // (importFromNickel enrichedFlakeInputs system baseDir (builtins.fromJSON
         (builtins.unsafeDiscardStringContext (builtins.readFile nickelResult))));
 in {
   inherit

lib/lockfile.ncl

@@ -0,0 +1,8 @@
+let filegen = import "files.ncl" in
+let nix = import "nix-interop/nix.ncl" in
+{
+  Schema = filegen.Schema,
+  config | Schema = {
+      files."nickel.lock.ncl".file = nix.import_nix "%%organist_internal#nickelLock",
+    }
+}

lib/modules/main.ncl

@@ -3,11 +3,21 @@
   T
     | doc m%"
       The type of a module.
+
+      A module is a record whose contract can be extended through merging.
     "%
     = {
-      config | Schema = {},
+      config
+        | doc m%"
+          The content of the module.
+        "%
+        | Schema
+        = {},
       Schema
         | not_exported
+        | doc m%"
+          The contract that is applied to `config`.
+        "%
         = {},
       "$__organist_type" | force = "module",
     },

lib/organist.ncl

@@ -1,13 +1,65 @@
 {
-  nix = import "./nix-interop/nix.ncl",
-  modules = import "./modules/main.ncl",
-  shells = nix.shells,
-  schema = import "./schema.ncl",
-  OrganistExpression = schema.OrganistExpression,
-  import_nix = nix.builtins.import_nix,
+  nix
+    | doc m%"
+      Nix interoperability library.
+    "%
+    = import "./nix-interop/nix.ncl",
+  modules
+    | doc m%"
+      implementation of a module system for Nickel, inspired by the NixOS
+      module system.
+    "%
+    = import "./modules/main.ncl",
+  shells
+    | doc m%"
+      Generic shells defined by Organist.
 
-  services = import "./services.ncl",
+      These are meant to be used as a base for the environments, and be combined or extended:
 
+      ### Example
+
+      ```nickel
+      {
+        shells = organist.shells.Bash,
+        shells = organist.shells.Rust,
+        shells.build.packages.pandoc = import_nix "nixpkgs#pandoc",
+      }
+      ```
+    "%
+    = nix.shells,
+  OrganistExpression
+    | doc m%"
+      The type of an Organist expression, defining the whole development environment.
+    "%
+    = (import "./schema.ncl").OrganistExpression,
+  import_nix
+  # XXX: That doc is duplicated from `nix-interop/builtins.ncl`.
+  # Keep it in sync, and remove once https://github.com/tweag/nickel/issues/1968 is fixed.
+    | doc m%%"
+      Import a Nix value from one of the flake inputs.
+      The value should be passed in a flakeref-like format
+      `input_name#attribute_path`.
+
+      # Example
+
+      ```nickel
+      cmd = nix-s%"%{import_nix "nixpkgs#hello"}/bin/hello > $out"%
+      ```
+    "%%
+    = nix.builtins.import_nix,
+
+  services
+    | doc m%"
+      Service management for Organist.
+
+      Merging this module with the configuration enables the `services` option.
+    "%
+    = import "./services.ncl",
+
+  tools
+    | doc m%"
+      Interface with various third-party tools.
+    "%,
   tools.editorconfig = import "./editorconfig.ncl",
   tools.direnv = import "./direnv.ncl",
 }

lib/schema.ncl

@@ -1,5 +1,6 @@
 let nix = import "./nix-interop/nix.ncl" in
 let filegen = import "files.ncl" in
+let lockfile = import "lockfile.ncl" in
 {
   OrganistShells = {
     dev
@@ -64,6 +65,11 @@ let filegen = import "files.ncl" in
           | FlakeOutputs
           = {},
       },
+      config | Schema = {
+          shells,
+          flake.devShells = shells,
+        },
     }
     & filegen
+    & lockfile
 }

project.ncl

@@ -70,6 +70,27 @@ organist.OrganistExpression
           touch $out
         "%,
           },
+
+          lsp = {
+            name = "organist-lsp-integration",
+            version = "0.0",
+            env.buildInputs = {
+              nls = import_nix "nixpkgs#nls",
+              python3 = import_nix "nixpkgs#python3",
+              pygls = import_nix "nixpkgs#python3Packages.pygls",
+              pytest = import_nix "nixpkgs#python3Packages.pytest",
+              pytest-asyncio = import_nix "nixpkgs#python3Packages.pytest-asyncio",
+            },
+            env = {
+              src = import_nix "self",
+              phases = ["unpackPhase", "testPhase", "installPhase"],
+              testPhase = nix-s%"
+                  cd tests/lsp
+                  pytest | tee $out
+                "%,
+              installPhase = "touch $out",
+            },
+          },
         },
 
       flake.checks = import "tests/main.ncl",

run-test.sh

@@ -51,28 +51,24 @@ test_one_template () (
   sed -i "s/shells\.Bash/shells.$target/" project.ncl
   prepare_shell
 
-  STORED_LOCKFILE_CONTENTS="$(cat nickel.lock.ncl)"
   TEST_SCRIPT="$(nickel export --format raw <<<'(import "'"$PROJECT_ROOT"'/lib/shell-tests.ncl").'"$target"'.script')"
 
   echo "Running with incorrect nickel.lock.ncl" 1>&2
+  echo '{}' > nickel.lock.ncl
   nix develop --accept-flake-config --print-build-logs --command bash <<<"$TEST_SCRIPT"
 
   if [[ $isFull == false ]]; then
     return
   fi
 
   echo "Running without nickel.lock.ncl" 1>&2
-  rm nickel.lock.ncl
+  rm -f nickel.lock.ncl
   nix develop --accept-flake-config --print-build-logs --command bash <<<"$TEST_SCRIPT"
 
   echo "Run with proper nickel.lock.ncl" 1>&2
-  nix run .\#regenerate-lockfile
-  PROPER_LOCKFILE_CONTENTS="$(cat nickel.lock.ncl)"
   nix develop --accept-flake-config --print-build-logs --command bash <<<"$TEST_SCRIPT"
 
   echo "Testing without flakes" 1>&2
-  # restore lockfile
-  cat > nickel.lock.ncl <<<"$STORED_LOCKFILE_CONTENTS"
   # pretend it's not flake anymore
   rm flake.*
   cat > shell.nix <<EOF
@@ -86,14 +82,15 @@ in
 EOF
 
   echo "Running with incorrect nickel.lock.ncl" 1>&2
+  rm -f nickel.lock.ncl
+  echo '{}' > nickel.lock.ncl
   nix develop --impure -f shell.nix -I nixpkgs="$NIXPKGS_PATH" --command bash <<<"$TEST_SCRIPT"
 
   echo "Running without nickel.lock.ncl" 1>&2
-  rm nickel.lock.ncl
+  rm -f nickel.lock.ncl
   nix develop --impure -f shell.nix -I nixpkgs="$NIXPKGS_PATH" --command bash <<<"$TEST_SCRIPT"
 
   echo "Run with proper nickel.lock.ncl" 1>&2
-  cat > nickel.lock.ncl <<<"$PROPER_LOCKFILE_CONTENTS"
   nix develop --impure -f shell.nix -I nixpkgs="$NIXPKGS_PATH" --command bash <<<"$TEST_SCRIPT"
 
   popd
@@ -120,7 +117,6 @@ test_example () (
   cp -r "$examplePath" ./example
   pushd ./example
   prepare_shell
-  nix run .\#regenerate-files --print-build-logs
   nix develop --print-build-logs --command bash test.sh
   popd
   popd