json-meta:// store

src/libstore/gc.cc

@@ -37,7 +37,7 @@ static std::string gcSocketPath = "/gc-socket/socket";
 static std::string gcRootsDir = "gcroots";
 
 
-void LocalStore::addIndirectRoot(const Path & path)
+void MixLocalStore::addIndirectRoot(const Path & path)
 {
     std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false);
     Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash));

src/libstore/json-meta-store.cc

@@ -0,0 +1,131 @@
+#include <fstream>
+#include <regex>
+#include <nlohmann/json.hpp>
+
+#include "archive.hh"
+#include "json-meta-store.hh"
+#include "callback.hh"
+#include "url.hh"
+#include "path-info.hh"
+#include "realisation.hh"
+
+namespace nix {
+
+std::string JsonMetaStoreConfig::doc()
+{
+    return
+        "#include json-meta-store.md"
+        ;
+}
+
+
+JsonMetaStore::JsonMetaStore(const Params & params)
+    : StoreConfig(params)
+    , LocalFSStoreConfig(params)
+    , JsonMetaStoreConfig(params)
+    , Store(params)
+    , LocalFSStore(params)
+{
+}
+
+
+JsonMetaStore::JsonMetaStore(
+    const std::string scheme,
+    std::string path,
+    const Params & params)
+    : JsonMetaStore(params)
+{
+    if (!path.empty())
+        throw UsageError("json-meta:// store url doesn't support path part, only scheme and query params");
+}
+
+
+std::string JsonMetaStore::getUri()
+{
+    return *uriSchemes().begin() + "://";
+}
+
+
+void JsonMetaStore::queryPathInfoUncached(
+    const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
+{
+    using nlohmann::json;
+
+    auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+    try
+    {
+        auto info_path = metaDir.get() + "/object/" + path.hashPart() + ".json";
+        try {
+            std::ifstream f { info_path };
+            auto json = json::parse(f);
+            auto info = std::make_shared<ValidPathInfo>(
+                path,
+                UnkeyedValidPathInfo::fromJSON(*this, json));
+            return (*callbackPtr)(std::move(info));
+        } catch (SysError &) {
+            return (*callbackPtr)({});
+        }
+    } catch (...) {
+        return callbackPtr->rethrow();
+    }
+}
+
+
+void JsonMetaStore::queryRealisationUncached(
+    const DrvOutput & drvOutput,
+    Callback<std::shared_ptr<const Realisation>> callback) noexcept
+{
+    using nlohmann::json;
+
+    auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+    try
+    {
+        auto realisation_path = metaDir.get() + "/realisation/" + drvOutput.to_string() + ".json";
+        try {
+            std::ifstream f { realisation_path };
+            auto json = json::parse(f);
+            auto realisation = std::make_shared<Realisation>(
+                Realisation::fromJSON(json, realisation_path));
+            return (*callbackPtr)(std::move(realisation));
+        } catch (SysError &) {
+            return (*callbackPtr)({});
+        }
+    } catch (...) {
+        return callbackPtr->rethrow();
+    }
+}
+
+
+// Unimplemented methods
+
+
+std::optional<StorePath> JsonMetaStore::queryPathFromHashPart(
+    const std::string & hashPart)
+{ unsupported("queryPathFromHashPart"); }
+
+
+void JsonMetaStore::addToStore(
+    const ValidPathInfo & info, Source & source,
+    RepairFlag repair, CheckSigsFlag checkSigs)
+{ unsupported("addToStore"); }
+
+
+StorePath JsonMetaStore::addTextToStore(
+    std::string_view name,
+    std::string_view s,
+    const StorePathSet & references,
+    RepairFlag repair)
+{ unsupported("addTextToStore"); }
+
+
+Roots JsonMetaStore::findRoots(bool censor)
+{ unsupported("findRoots"); }
+
+
+void JsonMetaStore::collectGarbage(const GCOptions & options, GCResults & results)
+{ unsupported("collectGarbage"); }
+
+
+static RegisterStoreImplementation<JsonMetaStore, JsonMetaStoreConfig> regJsonMetaStore;
+
+}

src/libstore/json-meta-store.hh

@@ -0,0 +1,70 @@
+#include "local-log-store.hh"
+
+namespace nix {
+
+/**
+ * Configuration for `JsonMetaStore`.
+ */
+struct JsonMetaStoreConfig : virtual LocalFSStoreConfig
+{
+    JsonMetaStoreConfig(const StringMap & params)
+        : StoreConfig(params)
+        , LocalFSStoreConfig(params)
+    {
+    }
+
+    const PathSetting metaDir{this,
+        rootDir.get() ? *rootDir.get() + "/nix/var/nix/metadata" : stateDir.get() + "/metadata",
+        "meta",
+        "directory where Nix will store metadata about store object."};
+
+    const std::string name() override { return "Experimental Local Cache Store"; }
+
+    std::string doc() override;
+};
+
+/**
+ * Local store that uses JSON files instead of a SQLite database.
+ */
+class JsonMetaStore
+    : public virtual JsonMetaStoreConfig
+    , public virtual MixLocalStore
+{
+
+public:
+    JsonMetaStore(const Params & params);
+    JsonMetaStore(const std::string scheme, std::string path, const Params & params);
+
+    std::string getUri() override;
+
+    static std::set<std::string> uriSchemes()
+    { return { "json-meta" }; }
+
+private:
+    // Overridden methods…
+
+    void queryPathInfoUncached(const StorePath & path,
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
+
+    void queryRealisationUncached(
+        const DrvOutput & drvOutput,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
+
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
+
+    void addToStore(
+        const ValidPathInfo & info, Source & source,
+        RepairFlag repair, CheckSigsFlag checkSigs) override;
+
+    StorePath addTextToStore(
+        std::string_view name,
+        std::string_view s,
+        const StorePathSet & references,
+        RepairFlag repair) override;
+
+    Roots findRoots(bool censor) override;
+
+    void collectGarbage(const GCOptions & options, GCResults & results) override;
+};
+
+}

src/libstore/json-meta-store.md

@@ -0,0 +1,15 @@
+R"(
+
+**Store URL format**: `json-meta://?root=*root*`
+
+This store type persists store objects on disk without an intervening daemon exactly like the the [Local Store] does.
+However, instead of storing store object metadata in a SQLite database, it stores it in JSON files in the [`meta`](...) directory.
+
+This is much less performant for many tasks, and not recommend for most users.
+However, it does have some benefits regarding synchronization and contention.
+For example, adding new store objects will not touch the JSON files for existing store objects in any way, whereas any change to the store at all with the [Local Store] will modify the SQLite database.
+Thus, for certain obscure use-cases of broadcasting a store to a wide number of consumers, it may be advantageous to use a store of this type instead of a Local Store.
+
+[Local Store]: ./local-store.md
+
+)"

src/libstore/local-log-store.cc

@@ -0,0 +1,31 @@
+#include "local-log-store.hh"
+#include "compression.hh"
+
+namespace nix {
+
+void MixLocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
+{
+    assert(drvPath.isDerivation());
+
+    auto baseName = drvPath.to_string();
+
+    auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
+
+    if (pathExists(logPath)) return;
+
+    createDirs(dirOf(logPath));
+
+    auto tmpFile = fmt("%s.tmp.%d", logPath, getpid());
+
+    writeFile(tmpFile, compress("bzip2", log));
+
+    std::filesystem::rename(tmpFile, logPath);
+}
+
+
+std::optional<TrustedFlag> MixLocalStore::isTrustedClient()
+{
+    return Trusted;
+}
+
+}

src/libstore/local-log-store.hh

@@ -0,0 +1,32 @@
+#pragma once
+///@file
+
+#include "indirect-root-store.hh"
+
+namespace nix {
+
+/**
+ * Store that directly manipulates the local log directory. Probably
+ * will evolve to be just anything a "true" local store (SQLite or JSON)
+ * has in common.
+ *
+ * @todo rename `LocalStore` to `SQLiteStore`, and then rename this to
+ * `MixLocalStore`. `LocalFSStore` could also be renamed to
+ * `MixFileSystemStore`.
+ */
+struct MixLocalStore : virtual IndirectRootStore {
+
+    /**
+     * Implementation of IndirectRootStore::addIndirectRoot().
+     *
+     * The weak reference merely is a symlink to `path' from
+     * /nix/var/nix/gcroots/auto/<hash of `path'>.
+     */
+    void addIndirectRoot(const Path & path) override;
+
+    void addBuildLog(const StorePath & drvPath, std::string_view log) override;
+
+    std::optional<TrustedFlag> isTrustedClient() override;
+};
+
+}

src/libstore/local-store.cc

@@ -1552,11 +1552,6 @@ unsigned int LocalStore::getProtocol()
     return PROTOCOL_VERSION;
 }
 
-std::optional<TrustedFlag> LocalStore::isTrustedClient()
-{
-    return Trusted;
-}
-
 
 void LocalStore::vacuumDB()
 {
@@ -1684,25 +1679,6 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id,
     }
 }
 
-void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
-{
-    assert(drvPath.isDerivation());
-
-    auto baseName = drvPath.to_string();
-
-    auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
-
-    if (pathExists(logPath)) return;
-
-    createDirs(dirOf(logPath));
-
-    auto tmpFile = fmt("%s.tmp.%d", logPath, getpid());
-
-    writeFile(tmpFile, compress("bzip2", log));
-
-    std::filesystem::rename(tmpFile, logPath);
-}
-
 std::optional<std::string> LocalStore::getVersion()
 {
     return nixVersion;

src/libstore/local-store.hh

@@ -6,6 +6,7 @@
 #include "pathlocks.hh"
 #include "store-api.hh"
 #include "indirect-root-store.hh"
+#include "local-log-store.hh"
 #include "sync.hh"
 
 #include <chrono>
@@ -76,6 +77,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
 class LocalStore : public virtual LocalStoreConfig
     , public virtual IndirectRootStore
     , public virtual GcStore
+    , public virtual MixLocalStore
 {
 private:
 
@@ -214,16 +216,6 @@ private:
      */
     Sync<AutoCloseFD> _fdRootsSocket;
 
-public:
-
-    /**
-     * Implementation of IndirectRootStore::addIndirectRoot().
-     *
-     * The weak reference merely is a symlink to `path' from
-     * /nix/var/nix/gcroots/auto/<hash of `path'>.
-     */
-    void addIndirectRoot(const Path & path) override;
-
 private:
 
     void findTempRoots(Roots & roots, bool censor);
@@ -310,8 +302,6 @@ public:
 
     unsigned int getProtocol() override;
 
-    std::optional<TrustedFlag> isTrustedClient() override;
-
     void vacuumDB();
 
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
@@ -401,8 +391,6 @@ private:
     void signPathInfo(ValidPathInfo & info);
     void signRealisation(Realisation &);
 
-    void addBuildLog(const StorePath & drvPath, std::string_view log) override;
-
     friend struct LocalDerivationGoal;
     friend struct PathSubstitutionGoal;
     friend struct SubstitutionGoal;