Add renderdoc API support

CMakeLists.txt

@@ -19,6 +19,7 @@ endif()
 
 project(Alber)
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
 
 if(APPLE)
     enable_language(OBJC)
@@ -46,6 +47,7 @@ option(ENABLE_QT_GUI "Enable the Qt GUI. If not selected then the emulator uses
 option(ENABLE_GIT_VERSIONING "Enables querying git for the emulator version" ON)
 option(BUILD_HYDRA_CORE "Build a Hydra core" OFF)
 option(BUILD_LIBRETRO_CORE "Build a Libretro core" OFF)
+option(ENABLE_RENDERDOC_API "Build with support for Renderdoc's capture API for graphics debugging" ON)
 
 if(BUILD_HYDRA_CORE)
     set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -134,6 +136,7 @@ add_subdirectory(third_party/toml11)
 include_directories(${SDL2_INCLUDE_DIR})
 include_directories(third_party/toml11)
 include_directories(third_party/glm)
+include_directories(third_party/renderdoc)
 
 add_subdirectory(third_party/cmrc)
 
@@ -192,6 +195,11 @@ else()
     set(HOST_ARM64 FALSE)
 endif()
 
+if(ENABLE_RENDERDOC_API)
+    find_package(RenderDoc 1.6.0 MODULE REQUIRED)
+    add_compile_definitions(PANDA3DS_ENABLE_RENDERDOC)
+endif()
+
 if(HOST_X64 OR HOST_ARM64)
     set(DYNARMIC_TESTS OFF)
     #set(DYNARMIC_NO_BUNDLED_FMT ON)
@@ -214,7 +222,7 @@ set(SOURCE_FILES src/emulator.cpp src/io_file.cpp src/config.cpp
                  src/core/CPU/cpu_dynarmic.cpp src/core/CPU/dynarmic_cycles.cpp
                  src/core/memory.cpp src/renderer.cpp src/core/renderer_null/renderer_null.cpp
                  src/http_server.cpp src/stb_image_write.c src/core/cheats.cpp src/core/action_replay.cpp
-                 src/discord_rpc.cpp src/lua.cpp src/memory_mapped_file.cpp src/miniaudio.cpp
+                 src/discord_rpc.cpp src/lua.cpp src/memory_mapped_file.cpp src/miniaudio.cpp src/renderdoc.cpp
 )
 set(CRYPTO_SOURCE_FILES src/core/crypto/aes_engine.cpp)
 set(KERNEL_SOURCE_FILES src/core/kernel/kernel.cpp src/core/kernel/resource_limits.cpp
@@ -292,7 +300,7 @@ set(HEADER_FILES include/emulator.hpp include/helpers.hpp include/termcolor.hpp
                  include/audio/miniaudio_device.hpp include/ring_buffer.hpp include/bitfield.hpp include/audio/dsp_shared_mem.hpp
                  include/audio/hle_core.hpp include/capstone.hpp include/audio/aac.hpp include/PICA/pica_frag_config.hpp
                  include/PICA/pica_frag_uniforms.hpp include/PICA/shader_gen_types.hpp include/PICA/shader_decompiler.hpp
-                 include/sdl_sensors.hpp
+                 include/sdl_sensors.hpp include/renderdoc.hpp
 )
 
 cmrc_add_resource_library(

cmake/FindRenderDoc.cmake

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+set(RENDERDOC_INCLUDE_DIR third_party/renderdoc)
+
+if (RENDERDOC_INCLUDE_DIR AND EXISTS "${RENDERDOC_INCLUDE_DIR}/renderdoc_app.h")
+    file(STRINGS "${RENDERDOC_INCLUDE_DIR}/renderdoc_app.h" RENDERDOC_VERSION_LINE REGEX "typedef struct RENDERDOC_API")
+    string(REGEX REPLACE ".*typedef struct RENDERDOC_API_([0-9]+)_([0-9]+)_([0-9]+).*" "\\1.\\2.\\3" RENDERDOC_VERSION "${RENDERDOC_VERSION_LINE}")
+    unset(RENDERDOC_VERSION_LINE)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(RenderDoc
+    REQUIRED_VARS RENDERDOC_INCLUDE_DIR
+    VERSION_VAR RENDERDOC_VERSION
+)
+
+if (RenderDoc_FOUND AND NOT TARGET RenderDoc::API)
+    add_library(RenderDoc::API INTERFACE IMPORTED)
+    set_target_properties(RenderDoc::API PROPERTIES
+        INTERFACE_INCLUDE_DIRECTORIES "${RENDERDOC_INCLUDE_DIR}"
+    )
+endif()
+
+mark_as_advanced(RENDERDOC_INCLUDE_DIR)

include/config.hpp

@@ -39,7 +39,8 @@ struct EmulatorConfig {
 
 	bool audioEnabled = false;
 	bool vsyncEnabled = true;
-
+
+	bool enableRenderdoc = false;
 	bool printAppVersion = true;
 	bool appVersionOnWindow = false;
 

include/emulator.hpp

@@ -135,4 +135,7 @@ class Emulator {
 	std::filesystem::path getAppDataRoot();
 
 	std::span<u8> getSMDH();
+
+  private:
+	void loadRenderdoc();
 };

include/renderdoc.hpp

@@ -0,0 +1,38 @@
+// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+#include <string>
+
+#include "helpers.hpp"
+
+#ifdef PANDA3DS_ENABLE_RENDERDOC
+namespace Renderdoc {
+	// Loads renderdoc dynamic library module.
+	void loadRenderdoc();
+
+	// Begins a capture if a renderdoc instance is attached.
+	void startCapture();
+
+	// Ends current renderdoc capture.
+	void endCapture();
+
+	// Triggers capturing process.
+	void triggerCapture();
+
+	// Sets output directory for captures
+	void setOutputDir(const std::string& path, const std::string& prefix);
+
+	// Returns whether we've compiled with Renderdoc support
+	static constexpr bool isSupported() { return true; }
+}  // namespace Renderdoc
+#else
+namespace Renderdoc {
+	static void loadRenderdoc() {}
+	static void startCapture() { Helpers::panic("Tried to start a Renderdoc capture while support for renderdoc is disabled") }
+	static void endCapture() { Helpers::panic("Tried to end a Renderdoc capture while support for renderdoc is disabled") }
+	static void triggerCapture() { Helpers::panic("Tried to trigger a Renderdoc capture while support for renderdoc is disabled") }
+	static void setOutputDir(const std::string& path, const std::string& prefix) {}
+	static constexpr bool isSupported() { return false; }
+}  // namespace Renderdoc
+#endif

src/config.cpp

@@ -70,6 +70,7 @@ void EmulatorConfig::load() {
 
 			forceShadergenForLights = toml::find_or<toml::boolean>(gpu, "ForceShadergenForLighting", true);
 			lightShadergenThreshold = toml::find_or<toml::integer>(gpu, "ShadergenLightThreshold", 1);
+			enableRenderdoc = toml::find_or<toml::boolean>(gpu, "EnableRenderdoc", false);
 		}
 	}
 
@@ -140,6 +141,7 @@ void EmulatorConfig::save() {
 	data["GPU"]["UseUbershaders"] = useUbershaders;
 	data["GPU"]["ForceShadergenForLighting"] = forceShadergenForLights;
 	data["GPU"]["ShadergenLightThreshold"] = lightShadergenThreshold;
+	data["GPU"]["EnableRenderdoc"] = enableRenderdoc;
 
 	data["Audio"]["DSPEmulation"] = std::string(Audio::DSPCore::typeToString(dspType));
 	data["Audio"]["EnableAudio"] = audioEnabled;

src/emulator.cpp

@@ -6,6 +6,8 @@
 
 #include <fstream>
 
+#include "renderdoc.hpp"
+
 #ifdef _WIN32
 #include <windows.h>
 
@@ -32,6 +34,10 @@ Emulator::Emulator()
 	audioDevice.init(dsp->getSamples());
 	setAudioEnabled(config.audioEnabled);
 
+	if (Renderdoc::isSupported() && config.enableRenderdoc) {
+		loadRenderdoc();
+	}
+
 #ifdef PANDA3DS_ENABLE_DISCORD_RPC
 	if (config.discordRpcEnabled) {
 		discordRpc.init();
@@ -431,3 +437,9 @@ void Emulator::setAudioEnabled(bool enable) {
 
 	dsp->setAudioEnabled(enable);
 }
+
+void Emulator::loadRenderdoc() {
+	std::string capturePath = (std::filesystem::current_path() / "RenderdocCaptures").generic_string();
+	Renderdoc::loadRenderdoc();
+	Renderdoc::setOutputDir(capturePath, "");
+}

src/panda_sdl/frontend_sdl.cpp

@@ -1,9 +1,10 @@
 #include "panda_sdl/frontend_sdl.hpp"
-#include "version.hpp"
 
 #include <glad/gl.h>
 
+#include "renderdoc.hpp"
 #include "sdl_sensors.hpp"
+#include "version.hpp"
 
 FrontendSDL::FrontendSDL() : keyboardMappings(InputMappings::defaultKeyboardMappings()) {
 	if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_EVENTS) < 0) {
@@ -140,6 +141,14 @@ void FrontendSDL::run() {
 								emu.reset(Emulator::ReloadOption::Reload);
 								break;
 							}
+
+							case SDLK_F11: {
+								if constexpr (Renderdoc::isSupported()) {
+									Renderdoc::triggerCapture();
+								}
+
+								break;
+							}
 						}
 					}
 					break;

src/renderdoc.cpp

@@ -0,0 +1,119 @@
+// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#ifdef PANDA3DS_ENABLE_RENDERDOC
+#include "renderdoc.hpp"
+
+#include <renderdoc_app.h>
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <dlfcn.h>
+#endif
+
+#include <array>
+#include <filesystem>
+
+namespace Renderdoc {
+	enum class CaptureState {
+		Idle,
+		Triggered,
+		InProgress,
+	};
+
+	static CaptureState captureState{CaptureState::Idle};
+	RENDERDOC_API_1_6_0* rdocAPI{};
+
+	void loadRenderdoc() {
+#ifdef WIN32
+		// Check if we are running in Renderdoc GUI
+		HMODULE mod = GetModuleHandleA("renderdoc.dll");
+		if (!mod) {
+			// If enabled in config, try to load RDoc runtime in offline mode
+			HKEY h_reg_key;
+			LONG result = RegOpenKeyExW(HKEY_LOCAL_MACHINE, L"SOFTWARE\\Classes\\RenderDoc.RDCCapture.1\\DefaultIcon\\", 0, KEY_READ, &h_reg_key);
+			if (result != ERROR_SUCCESS) {
+				return;
+			}
+			std::array<wchar_t, MAX_PATH> keyString{};
+			DWORD stringSize{keyString.size()};
+			result = RegQueryValueExW(h_reg_key, L"", 0, NULL, (LPBYTE)keyString.data(), &stringSize);
+			if (result != ERROR_SUCCESS) {
+				return;
+			}
+
+			std::filesystem::path path{keyString.cbegin(), keyString.cend()};
+			path = path.parent_path().append("renderdoc.dll");
+			const auto path_to_lib = path.generic_string();
+			mod = LoadLibraryA(path_to_lib.c_str());
+		}
+
+		if (mod) {
+			const auto RENDERDOC_GetAPI = reinterpret_cast<pRENDERDOC_GetAPI>(GetProcAddress(mod, "RENDERDOC_GetAPI"));
+			const s32 ret = RENDERDOC_GetAPI(eRENDERDOC_API_Version_1_6_0, (void**)&rdocAPI);
+
+			if (ret != 1) {
+				Helpers::panic("Invalid return value from RENDERDOC_GetAPI");
+			}
+		}
+#else
+#ifdef ANDROID
+		static constexpr const char RENDERDOC_LIB[] = "libVkLayer_GLES_RenderDoc.so";
+#else
+		static constexpr const char RENDERDOC_LIB[] = "librenderdoc.so";
+#endif
+		if (void* mod = dlopen(RENDERDOC_LIB, RTLD_NOW | RTLD_NOLOAD)) {
+			const auto RENDERDOC_GetAPI = reinterpret_cast<pRENDERDOC_GetAPI>(dlsym(mod, "RENDERDOC_GetAPI"));
+			const s32 ret = RENDERDOC_GetAPI(eRENDERDOC_API_Version_1_6_0, (void**)&rdocAPI);
+
+			if (ret != 1) {
+				Helpers::panic("Invalid return value from RENDERDOC_GetAPI");
+			}
+		}
+#endif
+		if (rdocAPI) {
+			// Disable default capture keys as they suppose to trigger present-to-present capturing
+			// and it is not what we want
+			rdocAPI->SetCaptureKeys(nullptr, 0);
+
+			// Also remove rdoc crash handler
+			rdocAPI->UnloadCrashHandler();
+		}
+	}
+
+	void startCapture() {
+		if (!rdocAPI) {
+			return;
+		}
+
+		if (captureState == CaptureState::Triggered) {
+			rdocAPI->StartFrameCapture(nullptr, nullptr);
+			captureState = CaptureState::InProgress;
+		}
+	}
+
+	void endCapture() {
+		if (!rdocAPI) {
+			return;
+		}
+
+		if (captureState == CaptureState::InProgress) {
+			rdocAPI->EndFrameCapture(nullptr, nullptr);
+			captureState = CaptureState::Idle;
+		}
+	}
+
+	void triggerCapture() {
+		if (captureState == CaptureState::Idle) {
+			captureState = CaptureState::Triggered;
+		}
+	}
+
+	void setOutputDir(const std::string& path, const std::string& prefix) {
+		if (rdocAPI) {
+			rdocAPI->SetCaptureFilePathTemplate((path + '\\' + prefix).c_str());
+		}
+	}
+}  // namespace Renderdoc
+#endif