Make the Derived Path family of types inductive for dynamic derivations

We want to be able to write down `foo.drv^bar.drv^baz`:
`foo.drv^bar.drv` is the dynamic derivation (since it is itself a
derivation output, `bar.drv` from `foo.drv`).

To that end, we create `Single{Derivation,BuiltPath}` types, that are
very similar except instead of having multiple outputs (in a set or
map), they have a single one. This is for everything to the left of the
rightmost `^`.

`NixStringContextElem` has an analogous change, and now can reuse
`SingleDerivedPath` at the top level. In fact, if we ever get rid of
`DrvDeep`, `NixStringContextElem` could be replaced with
`SingleDerivedPath` entirely!

Important note: some JSON formats have changed.

We already can *produce* dynamic derivations, but we can't refer to them
directly. Today, we can merely express building or example at the top
imperatively over time by building `foo.drv^bar.drv`, and then with a
second nix invocation doing `<result-from-first>^baz`, but this is not
declarative. The ethos of Nix of being able to write down the full plan
everything you want to do, and then execute than plan with a single
command, and for that we need the new inductive form of these types.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>

src/build-remote/build-remote.cc

@@ -322,7 +322,12 @@ static int main_build_remote(int argc, char * * argv)
                 throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
         } else {
             copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute);
-            auto res = sshStore->buildPathsWithResults({ DerivedPath::Built { *drvPath, OutputsSpec::All {} } });
+            auto res = sshStore->buildPathsWithResults({
+                DerivedPath::Built {
+                    .drvPath = staticDrvReq(*drvPath),
+                    .outputs = OutputsSpec::All {},
+                }
+            });
             // One path to build should produce exactly one build result
             assert(res.size() == 1);
             optResult = std::move(res[0]);

src/libcmd/built-path.cc

@@ -8,13 +8,39 @@
 
 namespace nix {
 
-nlohmann::json BuiltPath::Built::toJSON(ref<Store> store) const {
-    nlohmann::json res;
-    res["drvPath"] = store->printStorePath(drvPath);
-    for (const auto& [output, path] : outputs) {
-        res["outputs"][output] = store->printStorePath(path);
+#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \
+    bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
+    { \
+        const MY_TYPE* me = this; \
+        auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        me = &other; \
+        auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        return fields1 COMPARATOR fields2; \
     }
-    return res;
+#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
+    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \
+    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
+    CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
+
+#define FIELD_TYPE std::pair<std::string, StorePath>
+CMP(SingleBuiltPath, SingleBuiltPathBuilt, output)
+#undef FIELD_TYPE
+
+#define FIELD_TYPE std::map<std::string, StorePath>
+CMP(SingleBuiltPath, BuiltPathBuilt, outputs)
+#undef FIELD_TYPE
+
+#undef CMP
+#undef CMP_ONE
+
+StorePath SingleBuiltPath::outPath() const
+{
+    return std::visit(
+        overloaded{
+            [](const SingleBuiltPath::Opaque & p) { return p.path; },
+            [](const SingleBuiltPath::Built & b) { return b.output.second; },
+        }, raw()
+    );
 }
 
 StorePathSet BuiltPath::outPaths() const
@@ -32,6 +58,40 @@ StorePathSet BuiltPath::outPaths() const
     );
 }
 
+nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
+{
+    nlohmann::json res;
+    res["drvPath"] = drvPath->toJSON(store);
+    for (const auto & [outputName, outputPath] : outputs) {
+        res["outputs"][outputName] = store.printStorePath(outputPath);
+    }
+    return res;
+}
+
+nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const
+{
+    nlohmann::json res;
+    res["drvPath"] = drvPath->toJSON(store);
+    auto & [outputName, outputPath] = output;
+    res["output"] = outputName;
+    res["outputPath"] = store.printStorePath(outputPath);
+    return res;
+}
+
+nlohmann::json SingleBuiltPath::toJSON(const Store & store) const
+{
+    return std::visit([&](const auto & buildable) {
+        return buildable.toJSON(store);
+    }, raw());
+}
+
+nlohmann::json BuiltPath::toJSON(const Store & store) const
+{
+    return std::visit([&](const auto & buildable) {
+        return buildable.toJSON(store);
+    }, raw());
+}
+
 RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
 {
     RealisedPath::Set res;
@@ -40,15 +100,15 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
             [&](const BuiltPath::Opaque & p) { res.insert(p.path); },
             [&](const BuiltPath::Built & p) {
                 auto drvHashes =
-                    staticOutputHashes(store, store.readDerivation(p.drvPath));
+                    staticOutputHashes(store, store.readDerivation(p.drvPath->outPath()));
                 for (auto& [outputName, outputPath] : p.outputs) {
                     if (experimentalFeatureSettings.isEnabled(
                                 Xp::CaDerivations)) {
                         auto drvOutput = get(drvHashes, outputName);
                         if (!drvOutput)
                             throw Error(
                                 "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
-                                store.printStorePath(p.drvPath), outputName);
+                                store.printStorePath(p.drvPath->outPath()), outputName);
                         auto thisRealisation = store.queryRealisation(
                             DrvOutput{*drvOutput, outputName});
                         assert(thisRealisation);  // We’ve built it, so we must

src/libcmd/built-path.hh

@@ -2,19 +2,60 @@
 
 namespace nix {
 
+struct SingleBuiltPath;
+
+struct SingleBuiltPathBuilt {
+    ref<SingleBuiltPath> drvPath;
+    std::pair<std::string, StorePath> output;
+
+    std::string to_string(const Store & store) const;
+    static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
+    nlohmann::json toJSON(const Store & store) const;
+
+    DECLARE_CMP(SingleBuiltPathBuilt);
+};
+
+using _SingleBuiltPathRaw = std::variant<
+    DerivedPathOpaque,
+    SingleBuiltPathBuilt
+>;
+
+struct SingleBuiltPath : _SingleBuiltPathRaw {
+    using Raw = _SingleBuiltPathRaw;
+    using Raw::Raw;
+
+    using Opaque = DerivedPathOpaque;
+    using Built = SingleBuiltPathBuilt;
+
+    inline const Raw & raw() const {
+        return static_cast<const Raw &>(*this);
+    }
+
+    StorePath outPath() const;
+
+    static SingleBuiltPath parse(const Store & store, std::string_view);
+    nlohmann::json toJSON(const Store & store) const;
+};
+
+static inline ref<SingleBuiltPath> staticDrv(StorePath drvPath)
+{
+    return make_ref<SingleBuiltPath>(SingleBuiltPath::Opaque { drvPath });
+}
+
 /**
  * A built derived path with hints in the form of optional concrete output paths.
  *
  * See 'BuiltPath' for more an explanation.
  */
 struct BuiltPathBuilt {
-    StorePath drvPath;
+    ref<SingleBuiltPath> drvPath;
     std::map<std::string, StorePath> outputs;
 
-    nlohmann::json toJSON(ref<Store> store) const;
-    static BuiltPathBuilt parse(const Store & store, std::string_view);
+    std::string to_string(const Store & store) const;
+    static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
+    nlohmann::json toJSON(const Store & store) const;
 
-    GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs);
+    DECLARE_CMP(BuiltPathBuilt);
 };
 
 using _BuiltPathRaw = std::variant<
@@ -40,6 +81,7 @@ struct BuiltPath : _BuiltPathRaw {
     StorePathSet outPaths() const;
     RealisedPath::Set toRealisedPaths(Store & store) const;
 
+    nlohmann::json toJSON(const Store & store) const;
 };
 
 typedef std::vector<BuiltPath> BuiltPaths;

src/libcmd/installable-attr-path.cc

@@ -92,7 +92,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
     for (auto & [drvPath, outputs] : byDrvPath)
         res.push_back({
             .path = DerivedPath::Built {
-                .drvPath = drvPath,
+                .drvPath = staticDrvReq(drvPath),
                 .outputs = outputs,
             },
             .info = make_ref<ExtraPathInfoValue>(ExtraPathInfoValue::Value {

src/libcmd/installable-derived-path.cc

@@ -16,18 +16,24 @@ DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths()
     }};
 }
 
-std::optional<StorePath> InstallableDerivedPath::getStorePath()
+template<typename DP>
+static StorePath getStorePathFrom(const DP & derivedPath)
 {
     return std::visit(overloaded {
-        [&](const DerivedPath::Built & bfd) {
-            return bfd.drvPath;
+        [&](const typename DP::Built & bfd) {
+            return getStorePathFrom<SingleDerivedPath>(*bfd.drvPath);
         },
-        [&](const DerivedPath::Opaque & bo) {
+        [&](const typename DP::Opaque & bo) {
             return bo.path;
         },
     }, derivedPath.raw());
 }
 
+std::optional<StorePath> InstallableDerivedPath::getStorePath()
+{
+    return getStorePathFrom<DerivedPath>(derivedPath);
+}
+
 InstallableDerivedPath InstallableDerivedPath::parse(
     ref<Store> store,
     std::string_view prefix,
@@ -42,7 +48,7 @@ InstallableDerivedPath InstallableDerivedPath::parse(
             // Remove this prior to stabilizing the new CLI.
             if (storePath.isDerivation()) {
                 auto oldDerivedPath = DerivedPath::Built {
-                    .drvPath = storePath,
+                    .drvPath = staticDrvReq(storePath),
                     .outputs = OutputsSpec::All { },
                 };
                 warn(
@@ -55,8 +61,10 @@ InstallableDerivedPath InstallableDerivedPath::parse(
         },
         // If the user did use ^, we just do exactly what is written.
         [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath {
+            auto drv = make_ref<SingleDerivedPath>(SingleDerivedPath::parse(*store, prefix));
+            drvRequireExperiment(*drv);
             return DerivedPath::Built {
-                .drvPath = store->parseStorePath(prefix),
+                .drvPath = std::move(drv),
                 .outputs = outputSpec,
             };
         },

src/libcmd/installable-flake.cc

@@ -118,7 +118,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
 
     return {{
         .path = DerivedPath::Built {
-            .drvPath = std::move(drvPath),
+            .drvPath = staticDrvReq(std::move(drvPath)),
             .outputs = std::visit(overloaded {
                 [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
                     std::set<std::string> outputsToInstall;

src/libcmd/installable-value.cc

@@ -55,7 +55,8 @@ std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths
 
     else if (v.type() == nString) {
         return {{
-            .path = state->coerceToDerivedPath(pos, v, errorCtx),
+            .path = DerivedPath::fromSingle(
+                state->coerceToSingleDerivedPath(pos, v, errorCtx)),
             .info = make_ref<ExtraPathInfo>(),
         }};
     }

src/libcmd/installables.cc

@@ -514,6 +514,61 @@ ref<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
+static StorePath getBuiltPathFromRealisation(
+    Store & store,
+    const std::map<std::string, Hash> & outputHashes,
+    const DerivationOutputsAndOptPaths & drvOutputs,
+    const StorePath & drvPath,
+    const std::string & output)
+{
+    if (!outputHashes.count(output))
+        throw Error(
+            "the derivation '%s' doesn't have an output named '%s'",
+            store.printStorePath(drvPath), output);
+    if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
+        auto outputId =
+            DrvOutput{outputHashes.at(output), output};
+        auto realisation =
+            store.queryRealisation(outputId);
+        if (!realisation)
+            throw Error(
+                "cannot operate on an output of unbuilt "
+                "content-addressed derivation '%s'",
+                outputId.to_string());
+        return realisation->outPath;
+    } else {
+        // If ca-derivations isn't enabled, assume that
+        // the output path is statically known.
+        assert(drvOutputs.count(output));
+        assert(drvOutputs.at(output).second);
+        return *drvOutputs.at(output).second;
+    }
+}
+
+static SingleBuiltPath getBuiltPath(ref<Store> evalStore, ref<Store> store, const SingleDerivedPath & b)
+{
+    return std::visit(
+        overloaded{
+            [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath {
+                return SingleBuiltPath::Opaque { bo.path };
+            },
+            [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath {
+                auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath);
+                auto drv = evalStore->readDerivation(drvPath.outPath());
+                auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
+                auto drvOutputs = drv.outputsAndOptPaths(*store);
+                return SingleBuiltPath::Built {
+                    .drvPath = make_ref<SingleBuiltPath>(std::move(drvPath)),
+                    .output = {
+                        bfd.output,
+                        getBuiltPathFromRealisation(*store, outputHashes, drvOutputs, drvPath.outPath(), bfd.output),
+                    },
+                };
+            },
+        },
+        b.raw());
+}
+
 std::vector<BuiltPathWithResult> Installable::build(
     ref<Store> evalStore,
     ref<Store> store,
@@ -567,7 +622,10 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
                     [&](const DerivedPath::Built & bfd) {
                         auto outputs = resolveDerivedPath(*store, bfd, &*evalStore);
                         res.push_back({aux.installable, {
-                            .path = BuiltPath::Built { bfd.drvPath, outputs },
+                            .path = BuiltPath::Built {
+                                .drvPath = make_ref<SingleBuiltPath>(getBuiltPath(evalStore, store, *bfd.drvPath)),
+                                .outputs = outputs,
+                             },
                             .info = aux.info}});
                     },
                     [&](const DerivedPath::Opaque & bo) {
@@ -596,7 +654,10 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
                         for (auto & [outputName, realisation] : buildResult.builtOutputs)
                             outputs.emplace(outputName, realisation.outPath);
                         res.push_back({aux.installable, {
-                            .path = BuiltPath::Built { bfd.drvPath, outputs },
+                            .path = BuiltPath::Built {
+                                .drvPath = make_ref<SingleBuiltPath>(getBuiltPath(evalStore, store, *bfd.drvPath)),
+                                .outputs = outputs,
+                            },
                             .info = aux.info,
                             .result = buildResult}});
                     },
@@ -690,7 +751,14 @@ StorePathSet Installable::toDerivations(
                         : throw Error("argument '%s' did not evaluate to a derivation", i->what()));
                 },
                 [&](const DerivedPath::Built & bfd) {
-                    drvPaths.insert(bfd.drvPath);
+                    drvPaths.insert(std::visit(overloaded {
+                        [&](const SingleDerivedPath::Opaque o) -> StorePath {
+                            return o.path;
+                        },
+                        [&](const SingleDerivedPath::Built b) -> StorePath {
+                            throw Error("argument '%s' did not evaluate to output of dynamic derivation '%s' which is not yet built.", i->what(), b.to_string(*store));
+                        },
+                    }, tryResolveDerivedPath(*store, *bfd.drvPath).raw()));
                 },
             }, b.path.raw());
 

src/libcmd/repl.cc

@@ -647,7 +647,7 @@ bool NixRepl::processLine(std::string line)
         if (command == ":b" || command == ":bl") {
             state->store->buildPaths({
                 DerivedPath::Built {
-                    .drvPath = drvPath,
+                    .drvPath = staticDrvReq(drvPath),
                     .outputs = OutputsSpec::All { },
                 },
             });

src/libexpr/eval-cache.cc

@@ -599,7 +599,10 @@ string_t AttrCursor::getStringWithContext()
                             return d.drvPath;
                         },
                         [&](const NixStringContextElem::Built & b) -> const StorePath & {
-                            return b.drvPath;
+                            if (auto * drvPath = std::get_if<SingleDerivedPath::Opaque>(&*b.drvPath))
+                                return drvPath->path;
+                            else
+                                root->state.error("Dynamic derivations in string contexts are not yet supported by the evaluation cache").debugThrow<UnimplementedError>();
                         },
                         [&](const NixStringContextElem::Opaque & o) -> const StorePath & {
                             return o.path;