move templating code to lib file

flake.nix

@@ -88,13 +88,20 @@
                 mergeTests (importFiles [
                   ./test/modules/arr.nix
                   ./test/modules/davfs.nix
+                  ./test/modules/lib.nix
                   ./test/modules/nginx.nix
                   ./test/modules/postgresql.nix
                 ]);
             };
+
+            lib = nix-flake-tests.lib.check {
+              inherit pkgs;
+              tests = pkgs.callPackage ./test/modules/lib.nix {};
+            };
           }
           // (vm_test "authelia" ./test/vm/authelia.nix)
           // (vm_test "ldap" ./test/vm/ldap.nix)
+          // (vm_test "lib" ./test/vm/lib.nix)
           // (vm_test "postgresql" ./test/vm/postgresql.nix)
           // (vm_test "monitoring" ./test/vm/monitoring.nix)
           // (vm_test "nextcloud" ./test/vm/nextcloud.nix)

lib/default.nix

@@ -1,13 +1,114 @@
-{ lib }:
-{
-  template = file: newPath: replacements:
+{ pkgs, lib }:
+rec {
+  replaceSecrets = { userConfig, resultPath, generator }:
     let
-      templatePath = newPath + ".template";
+      configWithTemplates = withReplacements userConfig;
+
+      nonSecretConfigFile = pkgs.writeText "${resultPath}.template" (generator configWithTemplates);
+
+      replacements = getReplacements userConfig;
+    in
+      replaceSecretsScript {
+        file = nonSecretConfigFile;
+        inherit resultPath replacements;
+      };
+
+  template = file: newPath: replacements: replaceSecretsScript { inherit file replacements; resultPath = newPath; };
+  replaceSecretsScript = { file, resultPath, replacements }:
+    let
+      templatePath = resultPath + ".template";
       sedPatterns = lib.strings.concatStringsSep " " (lib.attrsets.mapAttrsToList (from: to: "-e \"s|${from}|${to}|\"") replacements);
     in
       ''
+      set -euo pipefail
+      set -x
       ln -fs ${file} ${templatePath}
-      rm ${newPath} || :
-      sed ${sedPatterns} ${templatePath} > ${newPath}
+      rm -f ${resultPath}
+      ${pkgs.gnused}/bin/sed ${sedPatterns} ${templatePath}
+      ${pkgs.gnused}/bin/sed ${sedPatterns} ${templatePath} > ${resultPath}
       '';
+
+  secretFileType = lib.types.submodule {
+    options = {
+      source = lib.mkOption {
+        type = lib.types.path;
+        description = "File containing the value.";
+      };
+
+      transform = lib.mkOption {
+        type = lib.types.raw;
+        description = "An optional function to transform the secret.";
+        default = null;
+        example = lib.literalExpression ''
+        v: "prefix-$${v}-suffix"
+        '';
+      };
+    };
+  };
+
+  secretName = name:
+      "%SECRET${lib.strings.toUpper (lib.strings.concatMapStrings (s: "_" + s) name)}%";
+
+  withReplacements = attrs:
+    let
+      valueOrReplacement = name: value:
+        if !(builtins.isAttrs value && value ? "source")
+        then value
+        else secretName name;
+    in
+      mapAttrsRecursiveCond (v: ! v ? "source") valueOrReplacement attrs;
+
+  getReplacements = attrs:
+    let
+      addNameField = name: value:
+        if !(builtins.isAttrs value && value ? "source")
+        then value
+        else value // { name = name; };
+
+      secretsWithName = mapAttrsRecursiveCond (v: ! v ? "source") addNameField attrs;
+
+      allSecrets = collect (v: builtins.isAttrs v && v ? "source") secretsWithName;
+
+      t = { transform ? null, ... }: if isNull transform then x: x else transform;
+
+      genReplacement = secret:
+        lib.attrsets.nameValuePair (secretName secret.name) ((t secret) "$(cat ${toString secret.source})");
+    in
+      lib.attrsets.listToAttrs (map genReplacement allSecrets);
+
+  # Like lib.attrsets.mapAttrsRecursiveCond but also recurses on lists.
+  mapAttrsRecursiveCond =
+    # A function, given the attribute set the recursion is currently at, determine if to recurse deeper into that attribute set.
+    cond:
+    # A function, given a list of attribute names and a value, returns a new value.
+    f:
+    # Attribute set to recursively map over.
+    set:
+    let
+      recurse = path:
+        let
+          g =
+            name: value:
+            if builtins.isAttrs value && cond value
+            then recurse (path ++ [name]) value
+            else if builtins.isList value
+            then lib.lists.imap0 (i: v: recurse (path ++ [name (builtins.toString i)]) v) value
+            else f (path ++ [name]) value;
+        in lib.attrsets.mapAttrs g;
+    in recurse [] set;
+
+  # Like lib.attrsets.collect but also recurses on lists.
+  collect =
+  # Given an attribute's value, determine if recursion should stop.
+  pred:
+  # The attribute set to recursively collect.
+  attrs:
+    if pred attrs then
+      [ attrs ]
+    else if builtins.isAttrs attrs then
+      lib.lists.concatMap (collect pred) (lib.attrsets.attrValues attrs)
+    else if builtins.isList attrs then
+      lib.lists.concatMap (collect pred) attrs
+    else
+      [];
 }

test/modules/lib.nix

@@ -0,0 +1,76 @@
+{ pkgs, lib, ... }:
+let
+  shblib = pkgs.callPackage ../../lib {};
+in
+{
+  # Tests that withReplacements can:
+  # - recurse in attrs and lists
+  # - .source field is understood
+  # - .transform field is understood
+  # - if .source field is found, ignores other fields
+  testLibWithReplacements = {
+    expected =
+      let
+        item = root: {
+          a = "A";
+          b = "%SECRET_${root}B%";
+          c = "%SECRET_${root}C%";
+        };
+      in
+        (item "") // {
+          nestedAttr = item "NESTEDATTR_";
+          nestedList = [ (item "NESTEDLIST_0_") ];
+          doubleNestedList = [ { n = (item "DOUBLENESTEDLIST_0_N_"); } ];
+        };
+    expr =
+      let
+        item = {
+          a = "A";
+          b.source = "/path/B";
+          b.transform = null;
+          c.source = "/path/C";
+          c.transform = v: "prefix-${v}-suffix";
+          c.other = "other";
+        };
+      in
+        shblib.withReplacements (
+          item // {
+            nestedAttr = item;
+            nestedList = [ item ];
+            doubleNestedList = [ { n = item; } ];
+          }
+        );
+  };
+
+  testLibGetReplacements = {
+    expected =
+      let
+        secrets = root: {
+          "%SECRET_${root}B%" = "$(cat /path/B)";
+          "%SECRET_${root}C%" = "prefix-$(cat /path/C)-suffix";
+        };
+      in
+        (secrets "") //
+        (secrets "NESTEDATTR_") //
+        (secrets "NESTEDLIST_0_") //
+        (secrets "DOUBLENESTEDLIST_0_N_");
+    expr =
+      let
+        item = {
+          a = "A";
+          b.source = "/path/B";
+          b.transform = null;
+          c.source = "/path/C";
+          c.transform = v: "prefix-${v}-suffix";
+          c.other = "other";
+        };
+      in
+        shblib.getReplacements (
+          item // {
+            nestedAttr = item;
+            nestedList = [ item ];
+            doubleNestedList = [ { n = item; } ];
+          }
+        );
+  };
+}

test/vm/lib.nix

@@ -0,0 +1,82 @@
+{ pkgs, lib, ... }:
+let
+  shblib = pkgs.callPackage ../../lib {};
+in
+{
+  template =
+    let
+      aSecret = pkgs.writeText "a-secret.txt" "Secret of A";
+      bSecret = pkgs.writeText "b-secret.txt" "Secret of B";
+      userConfig = {
+        a.a.source = aSecret;
+        b.source = bSecret;
+        b.transform = v: "prefix-${v}-suffix";
+        c = "not secret C";
+        d.d = "not secret D";
+      };
+
+      wantedConfig = {
+        a.a = "Secret of A";
+        b = "prefix-Secret of B-suffix";
+        c = "not secret C";
+        d.d = "not secret D";
+      };
+
+      configWithTemplates = shblib.withReplacements userConfig;
+
+      nonSecretConfigFile = pkgs.writeText "config.yaml.template" (lib.generators.toJSON {} configWithTemplates);
+
+      replacements = shblib.getReplacements userConfig;
+
+      replaceInTemplate = shblib.replaceSecretsScript {
+        file = nonSecretConfigFile;
+        resultPath = "/var/lib/config.yaml";
+        inherit replacements;
+      };
+
+      replaceInTemplate2 = shblib.replaceSecrets {
+        inherit userConfig;
+        resultPath = "/var/lib/config2.yaml";
+        generator = lib.generators.toJSON {};
+      };
+    in
+      pkgs.nixosTest {
+        name = "lib-template";
+        nodes.machine = { config, pkgs, ... }:
+          {
+            imports = [
+              {
+                options = {
+                  libtest.config = lib.mkOption {
+                    type = lib.types.attrsOf (lib.types.oneOf [ lib.types.str shblib.secretFileType ]);
+                  };
+                };
+              }
+            ];
+
+            system.activationScripts = {
+              libtest = replaceInTemplate;
+              libtest2 = replaceInTemplate2;
+            };
+          };
+
+        testScript = { nodes, ... }: ''
+        import json
+        start_all()
+
+        wantedConfig = json.loads('${lib.generators.toJSON {} wantedConfig}')
+        gotConfig = json.loads(machine.succeed("cat /var/lib/config.yaml"))
+        gotConfig2 = json.loads(machine.succeed("cat /var/lib/config2.yaml"))
+
+        # For debugging purpose
+        print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate" replaceInTemplate}"))
+        print(machine.succeed("cat ${pkgs.writeText "replaceInTemplate2" replaceInTemplate2}"))
+
+        if wantedConfig != gotConfig:
+          raise Exception("\nwantedConfig: {}\n!= gotConfig: {}".format(wantedConfig, gotConfig))
+
+        if wantedConfig != gotConfig2:
+          raise Exception("\nwantedConfig:  {}\n!= gotConfig2: {}".format(wantedConfig, gotConfig))
+        '';
+      };
+}