diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 2fb17d06f220..243df9d88984 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -322,7 +322,12 @@ static int main_build_remote(int argc, char * * argv) throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); - auto res = sshStore->buildPathsWithResults({ DerivedPath::Built { *drvPath, OutputsSpec::All {} } }); + auto res = sshStore->buildPathsWithResults({ + DerivedPath::Built { + .drvPath = staticDrvReq(*drvPath), + .outputs = OutputsSpec::All {}, + } + }); // One path to build should produce exactly one build result assert(res.size() == 1); optResult = std::move(res[0]); diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index db9c440e3df5..c6cc7fa9c75d 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -8,13 +8,39 @@ namespace nix { -nlohmann::json BuiltPath::Built::toJSON(ref store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = store->printStorePath(path); +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ } - return res; +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::pair +CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE std::map +CMP(SingleBuiltPath, BuiltPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +StorePath SingleBuiltPath::outPath() const +{ + return std::visit( + overloaded{ + [](const SingleBuiltPath::Opaque & p) { return p.path; }, + [](const SingleBuiltPath::Built & b) { return b.output.second; }, + }, raw() + ); } StorePathSet BuiltPath::outPaths() const @@ -32,6 +58,40 @@ StorePathSet BuiltPath::outPaths() const ); } +nlohmann::json BuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + for (const auto & [outputName, outputPath] : outputs) { + res["outputs"][outputName] = store.printStorePath(outputPath); + } + return res; +} + +nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + auto & [outputName, outputPath] = output; + res["output"] = outputName; + res["outputPath"] = store.printStorePath(outputPath); + return res; +} + +nlohmann::json SingleBuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +nlohmann::json BuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const { RealisedPath::Set res; @@ -40,7 +100,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Built & p) { auto drvHashes = - staticOutputHashes(store, store.readDerivation(p.drvPath)); + staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); for (auto& [outputName, outputPath] : p.outputs) { if (experimentalFeatureSettings.isEnabled( Xp::CaDerivations)) { @@ -48,7 +108,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const if (!drvOutput) throw Error( "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath), outputName); + store.printStorePath(p.drvPath->outPath()), outputName); auto thisRealisation = store.queryRealisation( DrvOutput{*drvOutput, outputName}); assert(thisRealisation); // We’ve built it, so we must diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index c563a46e9601..5efdb3eb984f 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -2,19 +2,60 @@ namespace nix { +struct SingleBuiltPath; + +struct SingleBuiltPathBuilt { + ref drvPath; + std::pair output; + + std::string to_string(const Store & store) const; + static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; + + DECLARE_CMP(SingleBuiltPathBuilt); +}; + +using _SingleBuiltPathRaw = std::variant< + DerivedPathOpaque, + SingleBuiltPathBuilt +>; + +struct SingleBuiltPath : _SingleBuiltPathRaw { + using Raw = _SingleBuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleBuiltPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + StorePath outPath() const; + + static SingleBuiltPath parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; +}; + +static inline ref staticDrv(StorePath drvPath) +{ + return make_ref(SingleBuiltPath::Opaque { drvPath }); +} + /** * A built derived path with hints in the form of optional concrete output paths. * * See 'BuiltPath' for more an explanation. */ struct BuiltPathBuilt { - StorePath drvPath; + ref drvPath; std::map outputs; - nlohmann::json toJSON(ref store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view); + std::string to_string(const Store & store) const; + static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; - GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); + DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -40,6 +81,7 @@ struct BuiltPath : _BuiltPathRaw { StorePathSet outPaths() const; RealisedPath::Set toRealisedPaths(Store & store) const; + nlohmann::json toJSON(const Store & store) const; }; typedef std::vector BuiltPaths; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index b35ca2910b2e..7fbdd451d0ad 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -92,7 +92,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() for (auto & [drvPath, outputs] : byDrvPath) res.push_back({ .path = DerivedPath::Built { - .drvPath = drvPath, + .drvPath = staticDrvReq(drvPath), .outputs = outputs, }, .info = make_ref(ExtraPathInfoValue::Value { diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index 6ecf54b7ce81..54f54e076ef5 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -16,18 +16,24 @@ DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() }}; } -std::optional InstallableDerivedPath::getStorePath() +template +static StorePath getStorePathFrom(const DP & derivedPath) { return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; + [&](const typename DP::Built & bfd) { + return getStorePathFrom(*bfd.drvPath); }, - [&](const DerivedPath::Opaque & bo) { + [&](const typename DP::Opaque & bo) { return bo.path; }, }, derivedPath.raw()); } +std::optional InstallableDerivedPath::getStorePath() +{ + return getStorePathFrom(derivedPath); +} + InstallableDerivedPath InstallableDerivedPath::parse( ref store, std::string_view prefix, @@ -42,7 +48,7 @@ InstallableDerivedPath InstallableDerivedPath::parse( // Remove this prior to stabilizing the new CLI. if (storePath.isDerivation()) { auto oldDerivedPath = DerivedPath::Built { - .drvPath = storePath, + .drvPath = staticDrvReq(storePath), .outputs = OutputsSpec::All { }, }; warn( @@ -55,8 +61,10 @@ InstallableDerivedPath InstallableDerivedPath::parse( }, // If the user did use ^, we just do exactly what is written. [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + auto drv = make_ref(SingleDerivedPath::parse(*store, prefix)); + drvRequireExperiment(*drv); return DerivedPath::Built { - .drvPath = store->parseStorePath(prefix), + .drvPath = std::move(drv), .outputs = outputSpec, }; }, diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 4da9b131b7e2..c93a5e567b29 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -118,7 +118,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() return {{ .path = DerivedPath::Built { - .drvPath = std::move(drvPath), + .drvPath = staticDrvReq(std::move(drvPath)), .outputs = std::visit(overloaded { [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { std::set outputsToInstall; diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 1eff293cc1a6..08ad35105bcf 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -55,7 +55,8 @@ std::optional InstallableValue::trySinglePathToDerivedPaths else if (v.type() == nString) { return {{ - .path = state->coerceToDerivedPath(pos, v, errorCtx), + .path = DerivedPath::fromSingle( + state->coerceToSingleDerivedPath(pos, v, errorCtx)), .info = make_ref(), }}; } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 10b077fb58c5..3f51906cad30 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -514,6 +514,61 @@ ref SourceExprCommand::parseInstallable( return installables.front(); } +static StorePath getBuiltPathFromRealisation( + Store & store, + const std::map & outputHashes, + const DerivationOutputsAndOptPaths & drvOutputs, + const StorePath & drvPath, + const std::string & output) +{ + if (!outputHashes.count(output)) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + store.printStorePath(drvPath), output); + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + auto outputId = + DrvOutput{outputHashes.at(output), output}; + auto realisation = + store.queryRealisation(outputId); + if (!realisation) + throw Error( + "cannot operate on an output of unbuilt " + "content-addressed derivation '%s'", + outputId.to_string()); + return realisation->outPath; + } else { + // If ca-derivations isn't enabled, assume that + // the output path is statically known. + assert(drvOutputs.count(output)); + assert(drvOutputs.at(output).second); + return *drvOutputs.at(output).second; + } +} + +static SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b) +{ + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { + return SingleBuiltPath::Opaque { bo.path }; + }, + [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { + auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); + auto drv = evalStore->readDerivation(drvPath.outPath()); + auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive + auto drvOutputs = drv.outputsAndOptPaths(*store); + return SingleBuiltPath::Built { + .drvPath = make_ref(std::move(drvPath)), + .output = { + bfd.output, + getBuiltPathFromRealisation(*store, outputHashes, drvOutputs, drvPath.outPath(), bfd.output), + }, + }; + }, + }, + b.raw()); +} + std::vector Installable::build( ref evalStore, ref store, @@ -567,7 +622,10 @@ std::vector, BuiltPathWithResult>> Installable::build [&](const DerivedPath::Built & bfd) { auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { @@ -596,7 +654,10 @@ std::vector, BuiltPathWithResult>> Installable::build for (auto & [outputName, realisation] : buildResult.builtOutputs) outputs.emplace(outputName, realisation.outPath); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info, .result = buildResult}}); }, @@ -690,7 +751,14 @@ StorePathSet Installable::toDerivations( : throw Error("argument '%s' did not evaluate to a derivation", i->what())); }, [&](const DerivedPath::Built & bfd) { - drvPaths.insert(bfd.drvPath); + drvPaths.insert(std::visit(overloaded { + [&](const SingleDerivedPath::Opaque o) -> StorePath { + return o.path; + }, + [&](const SingleDerivedPath::Built b) -> StorePath { + throw Error("argument '%s' did not evaluate to output of dynamic derivation '%s' which is not yet built.", i->what(), b.to_string(*store)); + }, + }, tryResolveDerivedPath(*store, *bfd.drvPath).raw())); }, }, b.path.raw()); diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index f9e9c2bf8f68..63e9f031ba5f 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -647,7 +647,7 @@ bool NixRepl::processLine(std::string line) if (command == ":b" || command == ":bl") { state->store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = staticDrvReq(drvPath), .outputs = OutputsSpec::All { }, }, }); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 9e734e654b2b..05ab91fdfc8b 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -599,7 +599,10 @@ string_t AttrCursor::getStringWithContext() return d.drvPath; }, [&](const NixStringContextElem::Built & b) -> const StorePath & { - return b.drvPath; + if (auto * drvPath = std::get_if(&*b.drvPath)) + return drvPath->path; + else + root->state.error("Dynamic derivations in string contexts are not yet supported by the evaluation cache").debugThrow(); }, [&](const NixStringContextElem::Opaque & o) -> const StorePath & { return o.path; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index be1bdb806f28..b055a03f8153 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1078,7 +1078,7 @@ void EvalState::mkOutputString( : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(), NixStringContext { NixStringContextElem::Built { - .drvPath = drvPath, + .drvPath = staticDrvReq(drvPath), .output = outputName, } }); @@ -2335,7 +2335,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon } -std::pair EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) +std::pair EvalState::coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) { NixStringContext context; auto s = forceString(v, context, pos, errorCtx); @@ -2346,21 +2346,16 @@ std::pair EvalState::coerceToDerivedPathUnchecked s, csize) .withTrace(pos, errorCtx).debugThrow(); auto derivedPath = std::visit(overloaded { - [&](NixStringContextElem::Opaque && o) -> DerivedPath { - return DerivedPath::Opaque { - .path = std::move(o.path), - }; + [&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { + return std::move(o); }, - [&](NixStringContextElem::DrvDeep &&) -> DerivedPath { + [&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath { error( "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time", s).withTrace(pos, errorCtx).debugThrow(); }, - [&](NixStringContextElem::Built && b) -> DerivedPath { - return DerivedPath::Built { - .drvPath = std::move(b.drvPath), - .outputs = OutputsSpec::Names { std::move(b.output) }, - }; + [&](NixStringContextElem::Built && b) -> SingleDerivedPath { + return std::move(b); }, }, ((NixStringContextElem &&) *context.begin()).raw()); return { @@ -2370,12 +2365,12 @@ std::pair EvalState::coerceToDerivedPathUnchecked } -DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) +SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) { - auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx); + auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx); auto s = s_; std::visit(overloaded { - [&](const DerivedPath::Opaque & o) { + [&](const SingleDerivedPath::Opaque & o) { auto sExpected = store->printStorePath(o.path); if (s != sExpected) error( @@ -2383,25 +2378,28 @@ DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::str s, sExpected) .withTrace(pos, errorCtx).debugThrow(); }, - [&](const DerivedPath::Built & b) { - // TODO need derived path with single output to make this - // total. Will add as part of RFC 92 work and then this is - // cleaned up. - auto output = *std::get(b.outputs).begin(); - - auto drv = store->readDerivation(b.drvPath); - auto i = drv.outputs.find(output); - if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output); - auto optOutputPath = i->second.path(*store, drv.name, output); - // This is testing for the case of CA derivations - auto sExpected = optOutputPath - ? store->printStorePath(*optOutputPath) - : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render(); + [&](const SingleDerivedPath::Built & b) { + auto placeholder = DownstreamPlaceholder::fromSingleDerivedPathBuilt(b); + auto sExpected = std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + auto drv = store->readDerivation(o.path); + auto i = drv.outputs.find(b.output); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output); + auto optOutputPath = i->second.path(*store, drv.name, b.output); + // This is testing for the case of CA derivations + return optOutputPath + ? store->printStorePath(*optOutputPath) + : placeholder.render(); + }, + [&](const SingleDerivedPath::Built & o) { + return placeholder.render(); + }, + }, b.drvPath->raw()); if (s != sExpected) error( "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", - s, output, store->printStorePath(b.drvPath), sExpected) + s, b.output, b.drvPath->to_string(*store), sExpected) .withTrace(pos, errorCtx).debugThrow(); } }, derivedPath.raw()); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 277e77ad59bd..1c5b5aebcf52 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -22,7 +22,7 @@ namespace nix { class Store; class EvalState; class StorePath; -struct DerivedPath; +struct SingleDerivedPath; enum RepairFlag : bool; @@ -532,12 +532,12 @@ public: StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); /** - * Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only. + * Part of `coerceToSingleDerivedPath()` without any store IO which is exposed for unit testing only. */ - std::pair coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); + std::pair coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); /** - * Coerce to `DerivedPath`. + * Coerce to `SingleDerivedPath`. * * Must be a string which is either a literal store path or a * "placeholder (see `DownstreamPlaceholder`). @@ -551,7 +551,7 @@ public: * source of truth, and ultimately tells us what we want, and then * we ensure the string corresponds to it. */ - DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); + SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); public: diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8a61e57cce9c..9381ae77a8ba 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -39,6 +39,20 @@ namespace nix { InvalidPathError::InvalidPathError(const Path & path) : EvalError("path '%s' is not valid", path), path(path) {} +// TODO dedup with copy-paste in installables.cc +template +static StorePath getStorePathFrom(const DP & derivedPath) +{ + return std::visit(overloaded { + [&](const typename DP::Built & bfd) { + return getStorePathFrom(*bfd.drvPath); + }, + [&](const typename DP::Opaque & bo) { + return bo.path; + }, + }, derivedPath.raw()); +} + StringMap EvalState::realiseContext(const NixStringContext & context) { std::vector drvs; @@ -55,7 +69,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) .drvPath = b.drvPath, .outputs = OutputsSpec::Names { b.output }, }); - ensureValid(b.drvPath); + ensureValid(getStorePathFrom(*b.drvPath)); }, [&](const NixStringContextElem::Opaque & o) { auto ctxS = store->printStorePath(o.path); @@ -76,7 +90,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) if (!evalSettings.enableImportFromDerivation) debugThrowLastTrace(Error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", - store->printStorePath(drvs.begin()->drvPath))); + drvs.begin()->to_string(*store))); /* Build/substitute the context. */ std::vector buildReqs; @@ -85,13 +99,15 @@ StringMap EvalState::realiseContext(const NixStringContext & context) /* Get all the output paths corresponding to the placeholders we had */ for (auto & drv : drvs) { - auto outputs = resolveDerivedPath(*store, drv); - for (auto & [outputName, outputPath] : outputs) { + for (auto & [outputName, outputPath] : resolveDerivedPath(*store, drv)) res.insert_or_assign( - DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = drv.drvPath, + .output = outputName, + }).render(), store->printStorePath(outputPath) ); - } } /* Add the output of this derivations to the allowed @@ -1251,7 +1267,10 @@ drvName, Bindings * attrs, Value & v) } }, [&](const NixStringContextElem::Built & b) { - drv.inputDrvs[b.drvPath].insert(b.output); + if (auto * p = std::get_if(&*b.drvPath)) + drv.inputDrvs[p->path].insert(b.output); + else + throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported"); }, [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 8b3468009bab..eb327684c428 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -106,7 +106,10 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, contextInfos[std::move(d.drvPath)].allOutputs = true; }, [&](NixStringContextElem::Built && b) { - contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output)); + // FIXME should eventually show string context as is, no + // resolving here. + auto drvPath = resolveDerivedPath(*state.store, *b.drvPath); + contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output)); }, [&](NixStringContextElem::Opaque && o) { contextInfos[std::move(o.path)].path = true; @@ -222,7 +225,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar for (auto elem : iter->value->listItems()) { auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); context.emplace(NixStringContextElem::Built { - .drvPath = namePath, + .drvPath = staticDrvReq(namePath), .output = std::string { outputName }, }); } diff --git a/src/libexpr/tests/derived-path.cc b/src/libexpr/tests/derived-path.cc index 8210efef22d0..bbec41b5b783 100644 --- a/src/libexpr/tests/derived-path.cc +++ b/src/libexpr/tests/derived-path.cc @@ -21,12 +21,12 @@ TEST_F(DerivedPathExpressionTest, force_init) RC_GTEST_FIXTURE_PROP( DerivedPathExpressionTest, prop_opaque_path_round_trip, - (const DerivedPath::Opaque & o)) + (const SingleDerivedPath::Opaque & o)) { auto * v = state.allocValue(); state.mkStorePathString(o.path, *v); - auto d = state.coerceToDerivedPath(noPos, *v, ""); - RC_ASSERT(DerivedPath { o } == d); + auto d = state.coerceToSingleDerivedPath(noPos, *v, ""); + RC_ASSERT(SingleDerivedPath { o } == d); } // TODO use DerivedPath::Built for parameter once it supports a single output @@ -39,12 +39,12 @@ RC_GTEST_FIXTURE_PROP( { auto * v = state.allocValue(); state.mkOutputString(*v, drvPath, outputName.name, std::nullopt); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + SingleDerivedPath::Built b { + .drvPath = staticDrvReq(drvPath), + .output = outputName.name, }; - RC_ASSERT(DerivedPath { b } == d); + RC_ASSERT(SingleDerivedPath { b } == d); } RC_GTEST_FIXTURE_PROP( @@ -54,12 +54,12 @@ RC_GTEST_FIXTURE_PROP( { auto * v = state.allocValue(); state.mkOutputString(*v, drvPath, outputName.name, outPath); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + SingleDerivedPath::Built b { + .drvPath = staticDrvReq(drvPath), + .output = outputName.name, }; - RC_ASSERT(DerivedPath { b } == d); + RC_ASSERT(SingleDerivedPath { b } == d); } } /* namespace nix */ diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 0d93815776fe..bf01a0174ca2 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -56,28 +56,50 @@ TEST(NixStringContextElemTest, drvDeep) { ASSERT_EQ(elem.to_string(), drvDeep); } -TEST(NixStringContextElemTest, built) { +TEST(NixStringContextElemTest, built_opaque) { std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(built); auto * p = std::get_if(&elem); ASSERT_TRUE(p); ASSERT_EQ(p->output, "foo"); - ASSERT_EQ(p->drvPath, StorePath { built.substr(5) }); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(5) }, + })); ASSERT_EQ(elem.to_string(), built); } +TEST(NixStringContextElemTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built, mockXpSettings); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->output, "foo"); + auto * drvPath = std::get_if(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "bar"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(9) }, + })); + ASSERT_EQ(elem.to_string(), built); } -namespace rc { -using namespace nix; +TEST(NixStringContextElemTest, built_built_xp) { + ASSERT_THROW( + NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); +} -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::Opaque { - .path = *gen::arbitrary(), - }); } +namespace rc { +using namespace nix; + Gen Arbitrary::arbitrary() { return gen::just(NixStringContextElem::DrvDeep { @@ -85,14 +107,6 @@ Gen Arbitrary::arb }); } -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::Built { - .drvPath = *gen::arbitrary(), - .output = (*gen::arbitrary()).name, - }); -} - Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, std::variant_size_v)) { diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index f76fc76e48d2..d8116011e132 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -4,29 +4,52 @@ namespace nix { -NixStringContextElem NixStringContextElem::parse(std::string_view s0) +NixStringContextElem NixStringContextElem::parse( + std::string_view s0, + const ExperimentalFeatureSettings & xpSettings) { std::string_view s = s0; + std::function parseRest; + parseRest = [&]() -> SingleDerivedPath { + // Case on whether there is a '!' + size_t index = s.find("!"); + if (index == std::string_view::npos) { + return SingleDerivedPath::Opaque { + .path = StorePath { s }, + }; + } else { + std::string output { s.substr(0, index) }; + // Advance string to parse after the '!' + s = s.substr(index + 1); + auto drv = make_ref(parseRest()); + drvRequireExperiment(*drv, xpSettings); + return SingleDerivedPath::Built { + .drvPath = std::move(drv), + .output = std::move(output), + }; + } + }; + if (s.size() == 0) { throw BadNixStringContextElem(s0, "String context element should never be an empty string"); } + switch (s.at(0)) { case '!': { - s = s.substr(1); // advance string to parse after first ! - size_t index = s.find("!"); - // This makes index + 1 safe. Index can be the length (one after index - // of last character), so given any valid character index --- a - // successful find --- we can add one. - if (index == std::string_view::npos) { + // Advance string to parse after the '!' + s = s.substr(1); + + // Find *second* '!' + if (s.find("!") == std::string_view::npos) { throw BadNixStringContextElem(s0, "String content element beginning with '!' should have a second '!'"); } - return NixStringContextElem::Built { - .drvPath = StorePath { s.substr(index + 1) }, - .output = std::string(s.substr(0, index)), - }; + + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } case '=': { return NixStringContextElem::DrvDeep { @@ -34,33 +57,51 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0) }; } default: { - return NixStringContextElem::Opaque { - .path = StorePath { s }, - }; + // Ensure no '!' + if (s.find("!") != std::string_view::npos) { + throw BadNixStringContextElem(s0, + "String content element not beginning with '!' should not have a second '!'"); + } + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } } } -std::string NixStringContextElem::to_string() const { - return std::visit(overloaded { +std::string NixStringContextElem::to_string() const +{ + std::string res; + + std::function toStringRest; + toStringRest = [&](auto & p) { + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + res += o.path.to_string(); + }, + [&](const SingleDerivedPath::Built & o) { + res += o.output; + res += '!'; + toStringRest(*o.drvPath); + }, + }, p.raw()); + }; + + std::visit(overloaded { [&](const NixStringContextElem::Built & b) { - std::string res; - res += '!'; - res += b.output; res += '!'; - res += b.drvPath.to_string(); - return res; + toStringRest(b); + }, + [&](const NixStringContextElem::Opaque & o) { + toStringRest(o); }, [&](const NixStringContextElem::DrvDeep & d) { - std::string res; res += '='; res += d.drvPath.to_string(); - return res; - }, - [&](const NixStringContextElem::Opaque & o) { - return std::string { o.path.to_string() }; }, }, raw()); + + return res; } } diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 287ae08a942d..a1b71695be95 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -3,7 +3,7 @@ #include "util.hh" #include "comparator.hh" -#include "path.hh" +#include "derived-path.hh" #include @@ -31,11 +31,7 @@ public: * * Encoded as just the path: ‘’. */ -struct NixStringContextElem_Opaque { - StorePath path; - - GENERATE_CMP(NixStringContextElem_Opaque, me->path); -}; +typedef SingleDerivedPath::Opaque NixStringContextElem_Opaque; /** * Path to a derivation and its entire build closure. @@ -57,12 +53,7 @@ struct NixStringContextElem_DrvDeep { * * Encoded in the form ‘!!’. */ -struct NixStringContextElem_Built { - StorePath drvPath; - std::string output; - - GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output); -}; +typedef SingleDerivedPath::Built NixStringContextElem_Built; using _NixStringContextElem_Raw = std::variant< NixStringContextElem_Opaque, @@ -93,8 +84,12 @@ struct NixStringContextElem : _NixStringContextElem_Raw { * - ‘’ * - ‘=’ * - ‘!!’ + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static NixStringContextElem parse(std::string_view s); + static NixStringContextElem parse( + std::string_view s, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); std::string to_string() const; }; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5e37f7ecbd1d..82e0cf92fd31 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -65,7 +65,7 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = staticDrvReq(drvPath), .outputs = wantedOutputs }) , useDerivation(true) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -74,7 +74,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store)); + DerivedPath::Built { staticDrvReq(drvPath), wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -84,7 +84,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = staticDrvReq(drvPath), .outputs = wantedOutputs }) , useDerivation(false) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -95,7 +95,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store)); + DerivedPath::Built { staticDrvReq(drvPath), drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -1490,7 +1490,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) for (auto & outputName : outputs->second) { auto buildResult = dg->getBuildResult(DerivedPath::Built { - .drvPath = dg->drvPath, + .drvPath = staticDrvReq(dg->drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 4aa4d6dcaf1e..d1c0a7edf8a5 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -77,7 +77,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat try { worker.run(Goals{goal}); return goal->getBuildResult(DerivedPath::Built { - .drvPath = drvPath, + .drvPath = staticDrvReq(drvPath), .outputs = OutputsSpec::All {}, }); } catch (Error & e) { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index ee66ee500118..e59126c6a356 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1170,6 +1170,19 @@ void LocalDerivationGoal::writeStructuredAttrs() } +static StorePath pathPartOfReq(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + return pathPartOfReq(*bfd.drvPath); + }, + }, req.raw()); +} + + static StorePath pathPartOfReq(const DerivedPath & req) { return std::visit(overloaded { @@ -1177,7 +1190,7 @@ static StorePath pathPartOfReq(const DerivedPath & req) return bo.path; }, [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; + return pathPartOfReq(*bfd.drvPath); }, }, req.raw()); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index a9ca9cbbc8c0..a30ccc0c2743 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -111,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode); + if (auto bop = std::get_if(&*bfd.drvPath)) + return makeDerivationGoal(bop->path, bfd.outputs, buildMode); + else + throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -265,7 +268,7 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs}); + topPaths.push_back(DerivedPath::Built{staticDrvReq(goal->drvPath), goal->wantedOutputs}); } else if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 52d073f8133c..605e05e14f6e 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -7,52 +7,135 @@ namespace nix { -nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ + } +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::string +CMP(SingleDerivedPath, SingleDerivedPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE OutputsSpec +CMP(SingleDerivedPath, DerivedPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const +{ + nlohmann::json res; + res["path"] = store.printStorePath(path); + return res; +} + +nlohmann::json SingleDerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["path"] = store->printStorePath(path); + res["drvPath"] = drvPath->toJSON(store); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); + res["output"] = output; + auto outputPathIter = outputMap.find(output); + if (outputPathIter == outputMap.end()) + res["outputPath"] = nullptr; + else if (std::optional p = outputPathIter->second) + res["outputPath"] = store.printStorePath(*p); + else + res["outputPath"] = nullptr; return res; } -nlohmann::json DerivedPath::Built::toJSON(ref store) const { +nlohmann::json DerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); + res["drvPath"] = drvPath->toJSON(store); // Fallback for the input-addressed derivation case: We expect to always be // able to print the output paths, so let’s do it - const auto outputMap = store->queryPartialDerivationOutputMap(drvPath); + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); for (const auto & [output, outputPathOpt] : outputMap) { if (!outputs.contains(output)) continue; if (outputPathOpt) - res["outputs"][output] = store->printStorePath(*outputPathOpt); + res["outputs"][output] = store.printStorePath(*outputPathOpt); else res["outputs"][output] = nullptr; } return res; } +nlohmann::json SingleDerivedPath::toJSON(Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +nlohmann::json DerivedPath::toJSON(Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + std::string DerivedPath::Opaque::to_string(const Store & store) const { return store.printStorePath(path); } +std::string SingleDerivedPath::Built::to_string(const Store & store) const +{ + return drvPath->to_string(store) + "^" + output; +} + +std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const +{ + return drvPath->to_string(store) + "!" + output; +} + std::string DerivedPath::Built::to_string(const Store & store) const { - return store.printStorePath(drvPath) + return drvPath->to_string(store) + '^' + outputs.to_string(); } std::string DerivedPath::Built::to_string_legacy(const Store & store) const { - return store.printStorePath(drvPath) - + '!' + return drvPath->to_string_legacy(store) + + "!" + outputs.to_string(); } +std::string SingleDerivedPath::to_string(const Store & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + std::string DerivedPath::to_string(const Store & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + +std::string SingleDerivedPath::to_string_legacy(const Store & store) const { return std::visit(overloaded { - [&](const DerivedPath::Built & req) { return req.to_string(store); }, - [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); }, + [&](const SingleDerivedPath::Opaque & req) { return req.to_string(store); }, }, this->raw()); } @@ -70,30 +153,123 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_ return {store.parseStorePath(s)}; } -DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS) +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings) +{ + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque &) { + // plain drv path; no experimental features required. + }, + [&](const SingleDerivedPath::Built &) { + xpSettings.require(Xp::DynamicDerivations); + }, + }, drv.raw()); +} + +SingleDerivedPath::Built SingleDerivedPath::Built::parse( + const Store & store, ref drv, + std::string_view output, + const ExperimentalFeatureSettings & xpSettings) { + drvRequireExperiment(*drv, xpSettings); return { - .drvPath = store.parseStorePath(drvS), + .drvPath = drv, + .output = std::string { output }, + }; +} + +DerivedPath::Built DerivedPath::Built::parse( + const Store & store, ref drv, + std::string_view outputsS, + const ExperimentalFeatureSettings & xpSettings) +{ + drvRequireExperiment(*drv, xpSettings); + return { + .drvPath = drv, .outputs = OutputsSpec::parse(outputsS), }; } -static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator) +static SingleDerivedPath parseWithSingle( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) +{ + size_t n = s.rfind(separator); + return n == s.npos + ? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s) + : (SingleDerivedPath) SingleDerivedPath::Built::parse(store, + make_ref(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) { - size_t n = s.find(separator); + return parseWithSingle(store, s, "^", xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "!", xpSettings); +} + +static DerivedPath parseWith( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) +{ + size_t n = s.rfind(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) - : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); + : (DerivedPath) DerivedPath::Built::parse(store, + make_ref(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); +} + +DerivedPath DerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWith(store, s, "^", xpSettings); } -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +DerivedPath DerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) { - return parseWith(store, s, "^"); + return parseWith(store, s, "!", xpSettings); } -DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) +DerivedPath DerivedPath::fromSingle(const SingleDerivedPath & req) { - return parseWith(store, s, "!"); + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) -> DerivedPath { + return o; + }, + [&](const SingleDerivedPath::Built & b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }; + }, + }, req.raw()); } } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 6ea80c92e511..95d7739ce76b 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -25,15 +25,110 @@ class Store; struct DerivedPathOpaque { StorePath path; - nlohmann::json toJSON(ref store) const; std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; GENERATE_CMP(DerivedPathOpaque, me->path); }; +struct SingleDerivedPath; + +/** + * A single derived path that is built from a derivation + * + * Built derived paths are pair of a derivation and an output name. They are + * evaluated by building the derivation, and then taking the resulting output + * path of the given output name. + */ +struct SingleDerivedPathBuilt { + ref drvPath; + std::string output; + + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPathBuilt parse( + const Store & store, ref drvPath, + std::string_view outputs, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; + + DECLARE_CMP(SingleDerivedPathBuilt); +}; + +using _SingleDerivedPathRaw = std::variant< + DerivedPathOpaque, + SingleDerivedPathBuilt +>; + +/** + * A "derived path" is a very simple sort of expression that evaluates + * to (concrete) store path. It is either: + * + * - opaque, in which case it is just a concrete store path with + * possibly no known derivation + * + * - built, in which case it is a pair of a derivation path and an + * output name. + */ +struct SingleDerivedPath : _SingleDerivedPathRaw { + using Raw = _SingleDerivedPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleDerivedPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + /** + * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; +}; + +static inline ref staticDrvReq(StorePath drvPath) +{ + return make_ref(SingleDerivedPath::Opaque { drvPath }); +} + /** - * A derived path that is built from a derivation + * A set of derived paths that are built from a derivation * * Built derived paths are pair of a derivation and some output names. * They are evaluated by building the derivation, and then replacing the @@ -45,7 +140,7 @@ struct DerivedPathOpaque { * output name. */ struct DerivedPathBuilt { - StorePath drvPath; + ref drvPath; OutputsSpec outputs; /** @@ -58,11 +153,16 @@ struct DerivedPathBuilt { std::string to_string_legacy(const Store & store) const; /** * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs); - nlohmann::json toJSON(ref store) const; + static DerivedPathBuilt parse( + const Store & store, ref, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; - GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); + DECLARE_CMP(DerivedPathBuilt); }; using _DerivedPathRaw = std::variant< @@ -72,13 +172,13 @@ using _DerivedPathRaw = std::variant< /** * A "derived path" is a very simple sort of expression that evaluates - * to (concrete) store path. It is either: + * to one or more (concrete) store paths. It is either: * - * - opaque, in which case it is just a concrete store path with + * - opaque, in which case it is just a single concrete store path with * possibly no known derivation * - * - built, in which case it is a pair of a derivation path and an - * output name. + * - built, in which case it is a pair of a derivation path and some + * output names. */ struct DerivedPath : _DerivedPathRaw { using Raw = _DerivedPathRaw; @@ -101,14 +201,43 @@ struct DerivedPath : _DerivedPathRaw { std::string to_string_legacy(const Store & store) const; /** * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parse(const Store & store, std::string_view); + static DerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parseLegacy(const Store & store, std::string_view); + static DerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convert a `SingleDerivedPath` to a `DerivedPath`. + */ + static DerivedPath fromSingle(const SingleDerivedPath &); + + nlohmann::json toJSON(Store & store) const; }; typedef std::vector DerivedPaths; +/** + * Used by various parser functions to require experimental features as + * needed. + * + * Somewhat unfortunate this cannot just be an implementation detail for + * this module. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); } diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 1752738f2fd7..2fed4876f5e9 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -36,4 +36,19 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( }; } +DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & b) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + return DownstreamPlaceholder::unknownCaOutput(o.path, b.output); + }, + [&](const SingleDerivedPath::Built & b2) { + return DownstreamPlaceholder::unknownDerivation( + DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2), + b.output); + }, + }, b.drvPath->raw()); +} + } diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh index f0c0dee773ee..a36119eec54d 100644 --- a/src/libstore/downstream-placeholder.hh +++ b/src/libstore/downstream-placeholder.hh @@ -3,6 +3,7 @@ #include "hash.hh" #include "path.hh" +#include "derived-path.hh" namespace nix { @@ -70,6 +71,16 @@ public: const DownstreamPlaceholder & drvPlaceholder, std::string_view outputName, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convenience constructor that handles both cases, delegating as + * needed. + * + * Recursively builds up a placeholder from a + * `SingleDerivedPath::Built.drvPath` chain. + */ + static DownstreamPlaceholder fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & built); }; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index fa17d606d430..78b05031ae69 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -358,6 +358,9 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor [&](const StorePath & drvPath) { throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); }, + [&](std::monostate) { + throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); + }, }, sOrDrvPath); } conn->to << ss; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 50336c77971e..17a42c8eda60 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -132,7 +132,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second })); + pool.enqueue(std::bind(doPath, DerivedPath::Built { staticDrvReq(i.first), i.second })); }; auto checkOutput = [&]( @@ -176,10 +176,18 @@ void Store::queryMissing(const std::vector & targets, std::visit(overloaded { [&](const DerivedPath::Built & bfd) { - if (!isValidPath(bfd.drvPath)) { + auto drvPathP = std::get_if(&*bfd.drvPath); + if (!drvPathP) { + // TODO make work in this case. + warn("Ignoring dynamic derivation %s; not yet implemented", bfd.drvPath->to_string(*this)); + return; + } + auto & drvPath = drvPathP->path; + + if (!isValidPath(drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(bfd.drvPath); + state->unknown.insert(drvPath); return; } @@ -187,7 +195,7 @@ void Store::queryMissing(const std::vector & targets, /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; - for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) { + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(drvPath)) { if (!pathOpt) { knownOutputPaths = false; break; @@ -197,15 +205,15 @@ void Store::queryMissing(const std::vector & targets, } if (knownOutputPaths && invalid.empty()) return; - auto drv = make_ref(derivationFromPath(bfd.drvPath)); - ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); + auto drv = make_ref(derivationFromPath(drvPath)); + ParsedDerivation parsedDrv(StorePath(drvPath), *drv); if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState)); } else - mustBuildDrv(bfd.drvPath, *drv); + mustBuildDrv(drvPath, *drv); }, [&](const DerivedPath::Opaque & bo) { @@ -313,7 +321,8 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, auto & evalStore = evalStore_ ? *evalStore_ : store; OutputPathMap outputs; - auto drv = evalStore.readDerivation(bfd.drvPath); + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + auto drv = evalStore.readDerivation(drvPath); auto outputHashes = staticOutputHashes(store, drv); auto drvOutputs = drv.outputsAndOptPaths(store); auto outputNames = std::visit(overloaded { @@ -332,7 +341,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, if (!outputHash) throw Error( "the derivation '%s' doesn't have an output named '%s'", - store.printStorePath(bfd.drvPath), output); + store.printStorePath(drvPath), output); if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { DrvOutput outputId { *outputHash, output }; auto realisation = store.queryRealisation(outputId); @@ -351,4 +360,95 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, return outputs; } + +SingleDerivedPath tryResolveDerivedPath(Store & store, const SingleDerivedPath & req, Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque &) -> SingleDerivedPath { + return req; + }, + [&](const SingleDerivedPath::Built & bfd0) -> SingleDerivedPath { + SingleDerivedPath::Built bfd { + make_ref(tryResolveDerivedPath(store, *bfd0.drvPath, evalStore_)), + bfd0.output, + }; + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> SingleDerivedPath { + auto & drvPath = bo.path; + auto outputPaths = evalStore.queryPartialDerivationOutputMap(drvPath); + if (outputPaths.count(bfd.output) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store.printStorePath(drvPath), bfd.output); + auto & optPath = outputPaths.at(bfd.output); + if (optPath) + // Can resolve this step + return DerivedPath::Opaque { *optPath }; + else + // Can't resolve this step + return bfd; + }, + [&](const SingleDerivedPath::Built & _) -> SingleDerivedPath { + // Can't resolve previous step, and thus all future steps. + return bfd; + }, + }, bfd.drvPath->raw()); + }, + }, req.raw()); +} + + +StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + auto outputPaths = evalStore.queryPartialDerivationOutputMap(drvPath); + if (outputPaths.count(bfd.output) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store.printStorePath(drvPath), bfd.output); + auto & optPath = outputPaths.at(bfd.output); + if (!optPath) + throw Error("'%s' does not yet map to a known concrete store path", + bfd.to_string(store)); + return *optPath; + }, + }, req.raw()); +} + + +OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) +{ + auto drvPath = resolveDerivedPath(store, *bfd.drvPath); + auto outputMap = store.queryDerivationOutputMap(drvPath); + auto outputsLeft = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return StringSet {}; + }, + [&](const OutputsSpec::Names & names) { + return static_cast(names); + }, + }, bfd.outputs.raw()); + for (auto iter = outputMap.begin(); iter != outputMap.end();) { + auto & outputName = iter->first; + if (bfd.outputs.contains(outputName)) { + outputsLeft.erase(outputName); + ++iter; + } else { + iter = outputMap.erase(iter); + } + } + if (!outputsLeft.empty()) + throw Error("derivation '%s' does not have an outputs %s", + store.printStorePath(drvPath), + concatStringsSep(", ", quoteStrings(std::get(bfd.outputs)))); + return outputMap; +} + } diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 869b490ade74..000427b45337 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -16,10 +16,16 @@ std::string StorePathWithOutputs::to_string(const Store & store) const DerivedPath StorePathWithOutputs::toDerivedPath() const { if (!outputs.empty()) { - return DerivedPath::Built { path, OutputsSpec::Names { outputs } }; + return DerivedPath::Built { + .drvPath = staticDrvReq(path), + .outputs = OutputsSpec::Names { outputs }, + }; } else if (path.isDerivation()) { assert(outputs.empty()); - return DerivedPath::Built { path, OutputsSpec::All { } }; + return DerivedPath::Built { + .drvPath = staticDrvReq(path), + .outputs = OutputsSpec::All { }, + }; } else { return DerivedPath::Opaque { path }; } @@ -34,29 +40,36 @@ std::vector toDerivedPaths(const std::vector } -std::variant StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) +StorePathWithOutputs::ParseResult StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) { return std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) -> std::variant { + [&](const DerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { if (bo.path.isDerivation()) { // drv path gets interpreted as "build", not "get drv file itself" return bo.path; } return StorePathWithOutputs { bo.path }; }, - [&](const DerivedPath::Built & bfd) -> std::variant { - return StorePathWithOutputs { - .path = bfd.drvPath, - // Use legacy encoding of wildcard as empty set - .outputs = std::visit(overloaded { - [&](const OutputsSpec::All &) -> StringSet { - return {}; - }, - [&](const OutputsSpec::Names & outputs) { - return static_cast(outputs); - }, - }, bfd.outputs.raw()), - }; + [&](const DerivedPath::Built & bfd) -> StorePathWithOutputs::ParseResult { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { + return StorePathWithOutputs { + .path = bo.path, + // Use legacy encoding of wildcard as empty set + .outputs = std::visit(overloaded { + [&](const OutputsSpec::All &) -> StringSet { + return {}; + }, + [&](const OutputsSpec::Names & outputs) { + return static_cast(outputs); + }, + }, bfd.outputs.raw()), + }; + }, + [&](const SingleDerivedPath::Built &) -> StorePathWithOutputs::ParseResult { + return std::monostate {}; + }, + }, bfd.drvPath->raw()); }, }, p.raw()); } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index d75850868104..57e03252df70 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -23,7 +23,9 @@ struct StorePathWithOutputs DerivedPath toDerivedPath() const; - static std::variant tryFromDerivedPath(const DerivedPath &); + typedef std::variant ParseResult; + + static StorePathWithOutputs::ParseResult tryFromDerivedPath(const DerivedPath &); }; std::vector toDerivedPaths(const std::vector); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1e2104e1faa5..ab589a4d0067 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -671,6 +671,9 @@ static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & con GET_PROTOCOL_MAJOR(conn.daemonVersion), GET_PROTOCOL_MINOR(conn.daemonVersion)); }, + [&](std::monostate) { + throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'"); + }, }, sOrDrvPath); } conn.to << ss; @@ -685,9 +688,23 @@ void RemoteStore::copyDrvsFromEvalStore( /* The remote doesn't have a way to access evalStore, so copy the .drvs. */ RealisedPath::Set drvPaths2; - for (auto & i : paths) - if (auto p = std::get_if(&i)) - drvPaths2.insert(p->drvPath); + for (const auto & i : paths) { + if (const auto * p = std::get_if(&i)) { + const SingleDerivedPath * sdp = &*p->drvPath; + bool cont = true; + while (cont) { + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bp) { + drvPaths2.insert(bp.path); + cont = false; + }, + [&](const SingleDerivedPath::Built & sdp2) { + sdp = &*sdp2.drvPath; + }, + }, sdp->raw()); + } + } + } copyClosure(*evalStore, *this, drvPaths2); } } @@ -757,7 +774,8 @@ std::vector RemoteStore::buildPathsWithResults( }; OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); + auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); + auto drv = evalStore->readDerivation(drvPath); const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto built = resolveDerivedPath(*this, bfd, &*evalStore); for (auto & [output, outputPath] : built) { @@ -765,7 +783,7 @@ std::vector RemoteStore::buildPathsWithResults( if (!outputHash) throw Error( "the derivation '%s' doesn't have an output named '%s'", - printStorePath(bfd.drvPath), output); + printStorePath(drvPath), output); auto outputId = DrvOutput{ *outputHash, output }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 14a862eefcad..42101f28c2f3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -915,10 +915,18 @@ void copyClosure( void removeTempRoots(); +/** + * Attempt to recursively replace derivation outputs with produced paths as + * much as possible, but where the derivation resolution doesn't exists, leave + * as-is. + */ +SingleDerivedPath tryResolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalStore = nullptr); + /** * Resolve the derived path completely, failing if any derivation output * is unknown. */ +StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalStore = nullptr); OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index 160443ec1d6a..d1de837bc448 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -17,14 +17,34 @@ Gen Arbitrary::arbitrary() }); } +Gen Arbitrary::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .output = (*gen::arbitrary()).name, + }); +} + Gen Arbitrary::arbitrary() { return gen::just(DerivedPath::Built { - .drvPath = *gen::arbitrary(), + .drvPath = make_ref(*gen::arbitrary()), .outputs = *gen::arbitrary(), }); } +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, std::variant_size_v)) { @@ -45,12 +65,69 @@ class DerivedPathTest : public LibStoreTest { }; -// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is -// no a real fixture. -// -// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args -TEST_F(DerivedPathTest, force_init) -{ +/** + * Round trip (string <-> data structure) test for + * `DerivedPath::Opaque`. + */ +TEST_F(DerivedPathTest, opaque) { + std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = DerivedPath::parse(*store, opaque); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->path, store->parseStorePath(opaque)); + ASSERT_EQ(elem.to_string(*store), opaque); +} + +/** + * Round trip (string <-> data structure) test for a simpler + * `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_opaque) { + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^bar,foo"; + auto elem = DerivedPath::parse(*store, built); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "foo", "bar" })); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Round trip (string <-> data structure) test for a a more complex, + * inductive `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"; + auto elem = DerivedPath::parse(*store, built, mockXpSettings); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "bar", "baz" })); + auto * drvPath = std::get_if(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "foo"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive derived path. + */ +TEST_F(DerivedPathTest, built_built_xp) { + ASSERT_THROW( + DerivedPath::parse(*store, "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"), + MissingExperimentalFeature); } RC_GTEST_FIXTURE_PROP( diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh index 506f3ccb1060..98d61f2283d2 100644 --- a/src/libstore/tests/derived-path.hh +++ b/src/libstore/tests/derived-path.hh @@ -12,8 +12,18 @@ namespace rc { using namespace nix; template<> -struct Arbitrary { - static Gen arbitrary(); +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); }; template<> diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 9f661c5c3986..7982fdc5e08a 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,6 +1,22 @@ #pragma once ///@file +/** + * Declare comparison methods without defining them. + */ +#define DECLARE_ONE_CMP(COMPARATOR, MY_TYPE) \ + bool operator COMPARATOR(const MY_TYPE & other) const; +#define DECLARE_EQUAL(my_type) \ + DECLARE_ONE_CMP(==, my_type) +#define DECLARE_LEQ(my_type) \ + DECLARE_ONE_CMP(<, my_type) +#define DECLARE_NEQ(my_type) \ + DECLARE_ONE_CMP(!=, my_type) +#define DECLARE_CMP(my_type) \ + DECLARE_EQUAL(my_type) \ + DECLARE_LEQ(my_type) \ + DECLARE_NEQ(my_type) + /** * Awful hacky generation of the comparison operators by doing a lexicographic * comparison between the choosen fields. diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 6510df8f041b..dd811bcbab16 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -393,7 +393,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { - .drvPath = bashDrv, + .drvPath = staticDrvReq(bashDrv), .outputs = OutputsSpec::Names {"out"}, }); pathsToCopy.insert(bashDrv); @@ -417,7 +417,7 @@ static void main_nix_build(int argc, char * * argv) })) { pathsToBuild.push_back(DerivedPath::Built { - .drvPath = inputDrv, + .drvPath = staticDrvReq(inputDrv), .outputs = OutputsSpec::Names { inputOutputs }, }); pathsToCopy.insert(inputDrv); @@ -590,7 +590,10 @@ static void main_nix_build(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}}); + pathsToBuild.push_back(DerivedPath::Built{ + .drvPath = staticDrvReq(drvPath), + .outputs = OutputsSpec::Names{outputName}, + }); pathsToBuildOrdered.push_back({drvPath, {outputName}}); drvsToCopy.insert(drvPath); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 91b073b49503..25b4863f55c8 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -481,7 +481,7 @@ static void printMissing(EvalState & state, DrvInfos & elems) for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) targets.push_back(DerivedPath::Built{ - .drvPath = *drvPath, + .drvPath = staticDrvReq(*drvPath), .outputs = OutputsSpec::All { }, }); else @@ -759,7 +759,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) std::vector paths { drvPath ? (DerivedPath) (DerivedPath::Built { - .drvPath = *drvPath, + .drvPath = staticDrvReq(*drvPath), .outputs = OutputsSpec::All { }, }) : (DerivedPath) (DerivedPath::Opaque { diff --git a/src/nix/app.cc b/src/nix/app.cc index e678b54f0f76..14a14cfeeeb3 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -24,7 +24,8 @@ StringPairs resolveRewrites( if (auto drvDep = std::get_if(&dep.path)) for (auto & [ outputName, outputPath ] : drvDep->outputs) res.emplace( - DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(), + DownstreamPlaceholder::unknownCaOutput( + drvDep->drvPath->outPath(), outputName).render(), store.printStorePath(outputPath) ); return res; @@ -64,7 +65,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath { /* We want all outputs of the drv */ return DerivedPath::Built { - .drvPath = d.drvPath, + .drvPath = staticDrvReq(d.drvPath), .outputs = OutputsSpec::All {}, }; }, @@ -105,7 +106,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) auto program = outPath + "/bin/" + mainProgram; return UnresolvedApp { App { .context = { DerivedPath::Built { - .drvPath = drvPath, + .drvPath = staticDrvReq(drvPath), .outputs = OutputsSpec::Names { outputName }, } }, .program = program, diff --git a/src/nix/build.cc b/src/nix/build.cc index ad1842a4ec48..479100186597 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -9,18 +9,18 @@ using namespace nix; -nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) +static nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, Store & store) { auto res = nlohmann::json::array(); for (auto & t : paths) { - std::visit([&res, store](const auto & t) { + std::visit([&](const auto & t) { res.push_back(t.toJSON(store)); }, t.raw()); } return res; } -nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, ref store) +static nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, const Store & store) { auto res = nlohmann::json::array(); for (auto & b : buildables) { @@ -125,7 +125,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile printMissing(store, pathsToBuild, lvlError); if (json) - logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + logger->cout("%s", derivedPathsToJSON(pathsToBuild, *store).dump()); return; } @@ -136,7 +136,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile installables, repair ? bmRepair : buildMode); - if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump()); + if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, *store).dump()); if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index bcc00d490dd4..6e3394f7e732 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -109,7 +109,7 @@ struct CmdBundle : InstallableValueCommand store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = staticDrvReq(drvPath), .outputs = OutputsSpec::All { }, }, }); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 195eeaa21e9a..9bbbc8451b76 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -235,7 +235,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore /* Build the derivation. */ store->buildPaths( { DerivedPath::Built { - .drvPath = shellDrvPath, + .drvPath = staticDrvReq(shellDrvPath), .outputs = OutputsSpec::All { }, }}, bmNormal, evalStore); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b5f5d0cac800..36536f0fbe74 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -543,9 +543,9 @@ struct CmdFlakeCheck : FlakeCommand *attr2.value, attr2.pos); if (drvPath && attr_name == settings.thisSystem.get()) { drvPaths.push_back(DerivedPath::Built { - .drvPath = *drvPath, - .outputs = OutputsSpec::All { }, - }); + .drvPath = staticDrvReq(*drvPath), + .outputs = OutputsSpec::All { }, + }); } } } diff --git a/src/nix/log.cc b/src/nix/log.cc index aaf82976404f..9a9bd30f9619 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -33,6 +33,17 @@ struct CmdLog : InstallableCommand auto b = installable->toDerivedPath(); + // For compat with CLI today, TODO revisit + auto oneUp = std::visit(overloaded { + [&](const DerivedPath::Opaque & bo) { + return make_ref(bo); + }, + [&](const DerivedPath::Built & bfd) { + return bfd.drvPath; + }, + }, b.path.raw()); + auto path = resolveDerivedPath(*store, *oneUp); + RunPager pager; for (auto & sub : subs) { auto * logSubP = dynamic_cast(&*sub); @@ -42,14 +53,7 @@ struct CmdLog : InstallableCommand } auto & logSub = *logSubP; - auto log = std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) { - return logSub.getBuildLog(bo.path); - }, - [&](const DerivedPath::Built & bfd) { - return logSub.getBuildLog(bfd.drvPath); - }, - }, b.path.raw()); + auto log = logSub.getBuildLog(path); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); diff --git a/tests/build-dry.sh b/tests/build-dry.sh index 6d1754af5c22..fe10fa08d354 100644 --- a/tests/build-dry.sh +++ b/tests/build-dry.sh @@ -56,12 +56,12 @@ RES=$(nix build -f dependencies.nix --dry-run --json) if [[ -z "${NIX_TESTS_CA_BY_DEFAULT-}" ]]; then echo "$RES" | jq '.[0] | [ - (.drvPath | test("'$NIX_STORE_DIR'.*\\.drv")), + (.drvPath.path | test("'$NIX_STORE_DIR'.*\\.drv")), (.outputs.out | test("'$NIX_STORE_DIR'")) ] | all' else echo "$RES" | jq '.[0] | [ - (.drvPath | test("'$NIX_STORE_DIR'.*\\.drv")), + (.drvPath.path | test("'$NIX_STORE_DIR'.*\\.drv")), .outputs.out == null ] | all' fi diff --git a/tests/build.sh b/tests/build.sh index 8ae20f0dff5e..be9540b802a8 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -5,13 +5,13 @@ clearStore # Make sure that 'nix build' returns all outputs by default. nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status ' (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and + (.drvPath.path | match(".*multiple-outputs-a.drv")) and (.outputs | (keys | length == 2) and (.first | match(".*multiple-outputs-a-first")) and (.second | match(".*multiple-outputs-a-second")))) and (.[1] | - (.drvPath | match(".*multiple-outputs-b.drv")) and + (.drvPath.path | match(".*multiple-outputs-b.drv")) and (.outputs | (keys | length == 1) and (.out | match(".*multiple-outputs-b")))) @@ -20,26 +20,26 @@ nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status ' # Test output selection using the '^' syntax. nix build -f multiple-outputs.nix --json a^first --no-link | jq --exit-status ' (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and + (.drvPath.path | match(".*multiple-outputs-a.drv")) and (.outputs | keys == ["first"])) ' nix build -f multiple-outputs.nix --json a^second,first --no-link | jq --exit-status ' (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and + (.drvPath.path | match(".*multiple-outputs-a.drv")) and (.outputs | keys == ["first", "second"])) ' nix build -f multiple-outputs.nix --json 'a^*' --no-link | jq --exit-status ' (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and + (.drvPath.path | match(".*multiple-outputs-a.drv")) and (.outputs | keys == ["first", "second"])) ' # Test that 'outputsToInstall' is respected by default. nix build -f multiple-outputs.nix --json e --no-link | jq --exit-status ' (.[0] | - (.drvPath | match(".*multiple-outputs-e.drv")) and + (.drvPath.path | match(".*multiple-outputs-e.drv")) and (.outputs | keys == ["a_a", "b"])) ' @@ -47,13 +47,13 @@ nix build -f multiple-outputs.nix --json e --no-link | jq --exit-status ' nix build -f multiple-outputs.nix --json e^a_a --no-link nix build -f multiple-outputs.nix --json e^a_a --no-link | jq --exit-status ' (.[0] | - (.drvPath | match(".*multiple-outputs-e.drv")) and + (.drvPath.path | match(".*multiple-outputs-e.drv")) and (.outputs | keys == ["a_a"])) ' nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status ' (.[0] | - (.drvPath | match(".*multiple-outputs-e.drv")) and + (.drvPath.path | match(".*multiple-outputs-e.drv")) and (.outputs | keys == ["a_a", "b", "c"])) ' @@ -61,7 +61,7 @@ nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status ' nix build -f multiple-outputs.nix --json 'e.a_a.outPath' --no-link | jq --exit-status ' (.[0] | - (.drvPath | match(".*multiple-outputs-e.drv")) and + (.drvPath.path | match(".*multiple-outputs-e.drv")) and (.outputs | keys == ["a_a"])) ' @@ -98,7 +98,7 @@ fi nix build "$drv^first" --no-link --json | jq --exit-status ' (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and + (.drvPath.path | match(".*multiple-outputs-a.drv")) and (.outputs | (keys | length == 1) and (.first | match(".*multiple-outputs-a-first")) and @@ -107,7 +107,7 @@ nix build "$drv^first" --no-link --json | jq --exit-status ' nix build "$drv^first,second" --no-link --json | jq --exit-status ' (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and + (.drvPath.path | match(".*multiple-outputs-a.drv")) and (.outputs | (keys | length == 2) and (.first | match(".*multiple-outputs-a-first")) and @@ -116,7 +116,7 @@ nix build "$drv^first,second" --no-link --json | jq --exit-status ' nix build "$drv^*" --no-link --json | jq --exit-status ' (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and + (.drvPath.path | match(".*multiple-outputs-a.drv")) and (.outputs | (keys | length == 2) and (.first | match(".*multiple-outputs-a-first")) and @@ -126,7 +126,7 @@ nix build "$drv^*" --no-link --json | jq --exit-status ' # Make sure that `--impure` works (regression test for https://github.com/NixOS/nix/issues/6488) nix build --impure -f multiple-outputs.nix --json e --no-link | jq --exit-status ' (.[0] | - (.drvPath | match(".*multiple-outputs-e.drv")) and + (.drvPath.path | match(".*multiple-outputs-e.drv")) and (.outputs | keys == ["a_a", "b"])) ' diff --git a/tests/test-libstoreconsumer/main.cc b/tests/test-libstoreconsumer/main.cc index 31b6d8ef16aa..8a961db5e37c 100644 --- a/tests/test-libstoreconsumer/main.cc +++ b/tests/test-libstoreconsumer/main.cc @@ -23,7 +23,7 @@ int main (int argc, char **argv) std::vector paths { DerivedPath::Built { - .drvPath = store->parseStorePath(drvPath), + .drvPath = staticDrvReq(store->parseStorePath(drvPath)), .outputs = OutputsSpec::Names{"out"} } };