Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: use portable C++ RNG #11627

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 0 additions & 9 deletions src/libmain/shared.cc
Original file line number Diff line number Diff line change
Expand Up @@ -174,15 +174,6 @@ void initNix(bool loadConfig)
everybody. */
umask(0022);

/* Initialise the PRNG. */
struct timeval tv;
gettimeofday(&tv, 0);
#ifndef _WIN32
srandom(tv.tv_usec);
#endif
srand(tv.tv_usec);


}


Expand Down
58 changes: 0 additions & 58 deletions src/libstore/build/derivation-goal.cc
Original file line number Diff line number Diff line change
Expand Up @@ -765,64 +765,6 @@ Goal::Co DerivationGoal::tryLocalBuild() {
}


static void chmod_(const Path & path, mode_t mode)
{
if (chmod(path.c_str(), mode) == -1)
throw SysError("setting permissions on '%s'", path);
}


/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
it's a directory and we're not root (to be able to update the
directory's parent link ".."). */
static void movePath(const Path & src, const Path & dst)
{
auto st = lstat(src);

bool changePerm = (
#ifndef _WIN32
geteuid()
#else
!isRootUser()
#endif
&& S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));

if (changePerm)
chmod_(src, st.st_mode | S_IWUSR);

std::filesystem::rename(src, dst);

if (changePerm)
chmod_(dst, st.st_mode);
}


void replaceValidPath(const Path & storePath, const Path & tmpPath)
{
/* We can't atomically replace storePath (the original) with
tmpPath (the replacement), so we have to move it out of the
way first. We'd better not be interrupted here, because if
we're repairing (say) Glibc, we end up with a broken system. */
Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), rand());
if (pathExists(storePath))
movePath(storePath, oldPath);

try {
movePath(tmpPath, storePath);
} catch (...) {
try {
// attempt to recover
movePath(oldPath, storePath);
} catch (...) {
ignoreExceptionExceptInterrupt();
}
throw;
}

deletePath(oldPath);
}


int DerivationGoal::getChildStatus()
{
#ifndef _WIN32 // TODO enable build hook on Windows
Expand Down
8 changes: 3 additions & 5 deletions src/libstore/filetransfer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "finally.hh"
#include "callback.hh"
#include "signals.hh"
#include "rng.hh"

#if ENABLE_S3
#include <aws/core/client/ClientConfiguration.h>
Expand All @@ -26,7 +27,6 @@
#include <cstring>
#include <iostream>
#include <queue>
#include <random>
#include <thread>
#include <regex>

Expand All @@ -42,8 +42,7 @@ struct curlFileTransfer : public FileTransfer
{
CURLM * curlm = 0;

std::random_device rd;
std::mt19937 mt19937;
RandomFloatGenerator rng{0.0, 1.0};

struct TransferItem : public std::enable_shared_from_this<TransferItem>
{
Expand Down Expand Up @@ -502,7 +501,7 @@ struct curlFileTransfer : public FileTransfer
|| writtenToSink == 0
|| (acceptRanges && encoding.empty())))
{
int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937));
int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + fileTransfer.rng());
if (writtenToSink)
warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
else
Expand Down Expand Up @@ -539,7 +538,6 @@ struct curlFileTransfer : public FileTransfer
std::thread workerThread;

curlFileTransfer()
: mt19937(rd())
{
static std::once_flag globalInit;
std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL);
Expand Down
2 changes: 1 addition & 1 deletion src/libstore/indirect-root-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ void IndirectRootStore::makeSymlink(const Path & link, const Path & target)
createDirs(dirOf(link));

/* Create the new symlink. */
Path tempLink = fmt("%1%.tmp-%2%-%3%", link, getpid(), rand());
Path tempLink = fmt("%1%.tmp-%2%-%3%", link, getpid(), rng());
createSymlink(target, tempLink);

/* Atomically replace the old one. */
Expand Down
3 changes: 3 additions & 0 deletions src/libstore/indirect-root-store.hh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
///@file

#include "local-fs-store.hh"
#include "rng.hh"

namespace nix {

Expand Down Expand Up @@ -68,6 +69,8 @@ struct IndirectRootStore : public virtual LocalFSStore
*/
virtual void addIndirectRoot(const Path & path) = 0;

RandomIntGenerator rng{};

protected:
void makeSymlink(const Path & link, const Path & target);
};
Expand Down
2 changes: 1 addition & 1 deletion src/libstore/optimise-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
its timestamp back to 0. */
MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : "");

std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), rand());
std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), rng());

try {
std::filesystem::create_hard_link(linkPath, tempLink);
Expand Down
3 changes: 2 additions & 1 deletion src/libstore/sqlite.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include "util.hh"
#include "url.hh"
#include "signals.hh"
#include "rng.hh"

#include <sqlite3.h>

Expand Down Expand Up @@ -258,7 +259,7 @@ void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning)
is likely to fail again. */
checkInterrupt();
/* <= 0.1s */
std::this_thread::sleep_for(std::chrono::milliseconds { rand() % 100 });
std::this_thread::sleep_for(std::chrono::milliseconds { RandomIntGenerator{0, 100}() });
}

}
33 changes: 29 additions & 4 deletions src/libstore/unix/build/local-derivation-goal.cc
Original file line number Diff line number Diff line change
Expand Up @@ -274,9 +274,11 @@ static void chmod_(const Path & path, mode_t mode)
}


/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
it's a directory and we're not root (to be able to update the
directory's parent link ".."). */
/**
* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
* it's a directory and we're not root (to be able to update the
* directory's parent link "..").
*/
static void movePath(const Path & src, const Path & dst)
{
auto st = lstat(src);
Expand All @@ -293,7 +295,30 @@ static void movePath(const Path & src, const Path & dst)
}


extern void replaceValidPath(const Path & storePath, const Path & tmpPath);
void LocalDerivationGoal::replaceValidPath(const Path & storePath, const Path & tmpPath)
{
/* We can't atomically replace storePath (the original) with
tmpPath (the replacement), so we have to move it out of the
way first. We'd better not be interrupted here, because if
we're repairing (say) Glibc, we end up with a broken system. */
Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), getLocalStore().rng());
Copy link
Member

@Mic92 Mic92 Oct 6, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't be part of this PR, but on linux, we can use renameat2 with RENAME_EXCHANGE to make the whole thing atomic without using a temporary directory.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you make a seperate issue for this?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if (pathExists(storePath))
movePath(storePath, oldPath);

try {
movePath(tmpPath, storePath);
} catch (...) {
try {
// attempt to recover
movePath(oldPath, storePath);
} catch (...) {
ignoreExceptionExceptInterrupt();
}
throw;
}

deletePath(oldPath);
}


int LocalDerivationGoal::getChildStatus()
Expand Down
2 changes: 2 additions & 0 deletions src/libstore/unix/build/local-derivation-goal.hh
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,8 @@ struct LocalDerivationGoal : public DerivationGoal
* rewrites caught everything
*/
StorePath makeFallbackPath(OutputNameView outputName);

void replaceValidPath(const Path & storePath, const Path & tmpPath);
};

}
1 change: 1 addition & 0 deletions src/libutil/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ headers = [config_h] + files(
'util.hh',
'variant-wrapper.hh',
'xml-writer.hh',
'rng.hh'
)

if host_machine.system() == 'linux'
Expand Down
40 changes: 40 additions & 0 deletions src/libutil/rng.hh
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#pragma once
/*! \file
* \brief Psuedo-random number generators class for easier use of C++'s random number generation facilities.
*/

#include <random>
#include <limits>

namespace nix {

// Inspired by the book "A Tour of C++, Third Edition" (ISBN-10 0136816487)
template<typename T, typename Distribution, typename Engine>
struct RandomNumberGenerator
Copy link
Member

@Mic92 Mic92 Oct 6, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a pseudo random generator, right? We should probably put this in the name so people not use it in the wrong place.

Copy link
Member Author

@bryanhonof bryanhonof Oct 7, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is any random number generator really a true random number generator? 😁
No, in all seriousness, I'm not sure. If people would want to use this for cryptographic reasons, I don't think this header is the right choice. Linking against openssl, and using its facilities, would be way better. But PseudoRandomNumberGenerator starts becoming very verbose as well. Maybe mention it in the comments of this header that it isn't meant for cryptographic reasons? Perhaps use Doxygen docs?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe mentioning in the header that this should not be used for cryptographic purpose would be a good start.

{
public:
using limits = std::numeric_limits<T>;
RandomNumberGenerator(T low = limits::min(), T high = limits::max())
: engine(std::random_device{}())
, dist(low, high){};
RandomNumberGenerator(std::seed_seq seed, T low, T high)
: engine(seed)
, dist(low, high){};
T operator()()
{
return dist(engine);
}
void seed(int s)
{
engine.seed(s);
}
private:
Engine engine;
Distribution dist;
};

using RandomIntGenerator = RandomNumberGenerator<int, std::uniform_int_distribution<int>, std::default_random_engine>;
using RandomFloatGenerator =
RandomNumberGenerator<float, std::uniform_real_distribution<float>, std::default_random_engine>;

} // namespace nix
Loading