From c54852f4f672e859f3ef6f7c63a6e856a55a6a8f Mon Sep 17 00:00:00 2001 From: Andreas Fredriksson Date: Sun, 4 Aug 2024 09:26:05 -0700 Subject: [PATCH 1/4] Remove warning about misleading indentation. --- src/Hash.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Hash.cpp b/src/Hash.cpp index 50dc4d58..aef06eee 100644 --- a/src/Hash.cpp +++ b/src/Hash.cpp @@ -65,8 +65,8 @@ void HashAddStringFoldCase(HashState* self, const char* path) char c = *path++; if (c == 0) return; - c = FoldCase(c); - HashUpdate(self, &c, 1); + c = FoldCase(c); + HashUpdate(self, &c, 1); } } From f3bb549a9475d1e0981af952a3dd4782c7ce3087 Mon Sep 17 00:00:00 2001 From: Andreas Fredriksson Date: Sun, 4 Aug 2024 09:41:40 -0700 Subject: [PATCH 2/4] Allow any number of build threads --- src/BuildQueue.cpp | 14 ++++++-------- src/BuildQueue.hpp | 9 ++------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/BuildQueue.cpp b/src/BuildQueue.cpp index ef7fbed8..1a059f5d 100644 --- a/src/BuildQueue.cpp +++ b/src/BuildQueue.cpp @@ -735,15 +735,10 @@ namespace t2 queue->m_ExpensiveWaitCount = 0; queue->m_ExpensiveWaitList = HeapAllocateArray(heap, capacity); - CHECK(queue->m_Queue); - - if (queue->m_Config.m_ThreadCount > kMaxBuildThreads) - { - Log(kWarning, "too many build threads (%d) - clamping to %d", - queue->m_Config.m_ThreadCount, kMaxBuildThreads); + queue->m_Threads = HeapAllocateArrayZeroed(config->m_Heap, config->m_ThreadCount); + queue->m_ThreadState = HeapAllocateArrayZeroed(config->m_Heap, config->m_ThreadCount); - queue->m_Config.m_ThreadCount = kMaxBuildThreads; - } + CHECK(queue->m_Queue); Log(kDebug, "build queue initialized; ring buffer capacity = %u", queue->m_QueueCapacity); @@ -797,6 +792,9 @@ namespace t2 CondDestroy(&queue->m_WorkAvailable); MutexDestroy(&queue->m_Lock); + HeapFree(config->m_Heap, queue->m_ThreadState); + HeapFree(config->m_Heap, queue->m_Threads); + // Unblock all signals on the main thread. SignalHandlerSetCondition(nullptr); SignalBlockThread(false); diff --git a/src/BuildQueue.hpp b/src/BuildQueue.hpp index e3e07dbc..fc96cb80 100644 --- a/src/BuildQueue.hpp +++ b/src/BuildQueue.hpp @@ -17,11 +17,6 @@ namespace t2 struct StatCache; struct DigestCache; - enum - { - kMaxBuildThreads = 64 - }; - struct BuildQueueConfig { enum @@ -73,8 +68,8 @@ namespace t2 int32_t m_PendingNodeCount; int32_t m_FailedNodeCount; int32_t m_CurrentPassIndex; - ThreadId m_Threads[kMaxBuildThreads]; - ThreadState m_ThreadState[kMaxBuildThreads]; + ThreadId *m_Threads; + ThreadState *m_ThreadState; int32_t m_ExpensiveRunning; int32_t m_ExpensiveWaitCount; NodeState **m_ExpensiveWaitList; From 76ca4e503fb8676f4cda912d17a18bd99f206ac7 Mon Sep 17 00:00:00 2001 From: Andreas Fredriksson Date: Sun, 4 Aug 2024 09:46:50 -0700 Subject: [PATCH 3/4] Use _SC_NPROCESSORS_ONLN to find online CPUs only Some Linux kernels are configured to allow "CPU hotswap" which means they pretend to have some huge array (128) of CPUs that might come online. This is weird. If we ask for the number of online CPUs that should be a saner value. std::thread::concurrency has the same issue, I think it uses the total count, which seems like a pretty bad bug in itself but outside the scope of this project. --- src/Common.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Common.cpp b/src/Common.cpp index 0a9a2bf4..9217c376 100644 --- a/src/Common.cpp +++ b/src/Common.cpp @@ -24,10 +24,6 @@ #include #endif -#if defined(TUNDRA_LINUX) -#include -#endif - #if defined(TUNDRA_WIN32) #include #include @@ -346,9 +342,7 @@ int GetCpuCount() GetSystemInfo(&si); return (int) si.dwNumberOfProcessors; #elif defined(TUNDRA_LINUX) - return (int)std::thread::hardware_concurrency(); -#else - long nprocs_max = sysconf(_SC_NPROCESSORS_CONF); + long nprocs_max = sysconf(_SC_NPROCESSORS_ONLN); if (nprocs_max < 0) CroakErrno("couldn't get CPU count"); return (int) nprocs_max; From 03ff325345ef88c54896d79815477a190389f482 Mon Sep 17 00:00:00 2001 From: Andreas Fredriksson Date: Sun, 4 Aug 2024 09:59:50 -0700 Subject: [PATCH 4/4] Allocate scratch handle array on Windows dynamically --- src/Exec.hpp | 2 +- src/ExecUnix.cpp | 2 +- src/ExecWin32.cpp | 5 +++-- src/Main.cpp | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Exec.hpp b/src/Exec.hpp index 9c5631b1..6755510c 100644 --- a/src/Exec.hpp +++ b/src/Exec.hpp @@ -15,7 +15,7 @@ namespace t2 bool m_WasSignalled; }; - void ExecInit(void); + void ExecInit(int thread_count); ExecResult ExecuteProcess( const char* cmd_line, diff --git a/src/ExecUnix.cpp b/src/ExecUnix.cpp index d152c4fb..f9ad43f3 100644 --- a/src/ExecUnix.cpp +++ b/src/ExecUnix.cpp @@ -37,7 +37,7 @@ static void SetFdNonBlocking(int fd) CroakErrno("couldn't unblock fd %d", fd); } -void ExecInit(void) +void ExecInit(int thread_count) { TerminalIoInit(); } diff --git a/src/ExecWin32.cpp b/src/ExecWin32.cpp index 5a0c054a..a1d16540 100644 --- a/src/ExecWin32.cpp +++ b/src/ExecWin32.cpp @@ -45,7 +45,7 @@ static char s_TemporaryDir[MAX_PATH]; static DWORD s_TundraPid; static Mutex s_FdMutex; -static HANDLE s_TempFiles[kMaxBuildThreads]; +static HANDLE *s_TempFiles; static HANDLE AllocFd(int job_id) { @@ -176,8 +176,9 @@ static char UTF8_WindowsEnvironment[128*1024]; static size_t g_Win32EnvCount; -void ExecInit(void) +void ExecInit(int thread_count) { + s_TempFiles = (HANDLE*) calloc(thread_count, sizeof s_TempFiles[0]); s_TundraPid = GetCurrentProcessId(); if (0 == GetTempPathA(sizeof(s_TemporaryDir), s_TemporaryDir)) diff --git a/src/Main.cpp b/src/Main.cpp index b20c0229..5834b5de 100644 --- a/src/Main.cpp +++ b/src/Main.cpp @@ -334,7 +334,7 @@ int main(int argc, char* argv[]) uint64_t start_time = TimerGet(); - ExecInit(); + ExecInit(options.m_ThreadCount); if (options.m_WorkingDir) {