feat: add RenderDocCapture to Debug module for frame capture debugging

- Add RenderDocCapture class for dynamic loading of renderdoc.dll
- Support BeginCapture/EndCapture/TriggerCapture APIs
- Add RenderDoc log category
- Add unit tests for RenderDocCapture in tests/debug

engine/CMakeLists.txt

@@ -80,6 +80,7 @@ add_library(XCEngine STATIC
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Debug/FileLogSink.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Debug/Logger.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Debug/Profiler.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/Debug/RenderDocCapture.cpp
 
     # RHI
     ${CMAKE_CURRENT_SOURCE_DIR}/include/XCEngine/RHI/RHIEnums.h

engine/include/XCEngine/Debug/Debug.h

@@ -8,5 +8,6 @@
 #include "FileLogSink.h"
 #include "Logger.h"
 #include "Profiler.h"
+#include "RenderDocCapture.h"
 
 #include "../Core/FileWriter.h"

engine/include/XCEngine/Debug/LogCategory.h

@@ -13,6 +13,7 @@ enum class LogCategory {
     Memory,
     Threading,
     FileSystem,
+    RenderDoc,
     Custom
 };
 

engine/include/XCEngine/Debug/RenderDocCapture.h

@@ -0,0 +1,76 @@
+#pragma once
+
+#include <cstdint>
+#include <windows.h>
+
+namespace XCEngine {
+namespace Debug {
+
+#define RENDERDOC_CC __cdecl
+
+class RenderDocCapture {
+public:
+    static RenderDocCapture& Get();
+
+    bool Initialize(void* device, void* window = nullptr);
+    void Shutdown();
+
+    bool IsLoaded() const { return m_isLoaded; }
+    bool IsCapturing() const;
+
+    void BeginCapture(const char* title = nullptr);
+    void EndCapture();
+    void TriggerCapture();
+
+    void SetCaptureFilePath(const char* path);
+    void SetCaptureComments(const char* comments);
+
+private:
+    RenderDocCapture() = default;
+    ~RenderDocCapture() = default;
+
+    bool LoadRenderDoc();
+    void UnloadRenderDoc();
+
+    struct RENDERDOC_API_1_7_0 {
+        void (RENDERDOC_CC* GetAPIVersion)(int* major, int* minor, int* patch);
+        void (RENDERDOC_CC* SetCaptureOptionU32)(uint32_t opt, uint32_t val);
+        void (RENDERDOC_CC* SetCaptureOptionF32)(uint32_t opt, float val);
+        uint32_t (RENDERDOC_CC* GetCaptureOptionU32)(uint32_t opt);
+        float (RENDERDOC_CC* GetCaptureOptionF32)(uint32_t opt);
+        void (RENDERDOC_CC* SetFocusToggleKeys)(const char* const* keys, int num);
+        void (RENDERDOC_CC* SetCaptureKeys)(const char* const* keys, int num);
+        uint32_t (RENDERDOC_CC* GetOverlayBits)();
+        void (RENDERDOC_CC* MaskOverlayBits)(uint32_t And, uint32_t Or);
+        void (RENDERDOC_CC* RemoveHooks)();
+        void (RENDERDOC_CC* UnloadCrashHandler)();
+        void (RENDERDOC_CC* SetCaptureFilePathTemplate)(const char* path);
+        const char* (RENDERDOC_CC* GetCaptureFilePathTemplate)();
+        uint32_t (RENDERDOC_CC* GetNumCaptures)();
+        uint32_t (RENDERDOC_CC* GetCapture)(uint32_t idx, char* filename, uint32_t* length, uint64_t* timestamp);
+        void (RENDERDOC_CC* TriggerCapture)();
+        uint32_t (RENDERDOC_CC* IsTargetControlConnected)();
+        void (RENDERDOC_CC* LaunchReplayUI)(uint32_t connect, const char* cmdline);
+        void (RENDERDOC_CC* SetActiveWindow)(void* device, void* window);
+        void (RENDERDOC_CC* StartFrameCapture)(void* device, void* window);
+        uint32_t (RENDERDOC_CC* IsFrameCapturing)();
+        void (RENDERDOC_CC* EndFrameCapture)(void* device, void* window);
+        void (RENDERDOC_CC* TriggerMultiFrameCapture)(uint32_t numFrames);
+        void (RENDERDOC_CC* SetCaptureFileComments)(const char* filePath, const char* comments);
+        uint32_t (RENDERDOC_CC* DiscardFrameCapture)(void* device, void* window);
+        uint32_t (RENDERDOC_CC* ShowReplayUI)();
+        void (RENDERDOC_CC* SetCaptureTitle)(const char* title);
+        void (RENDERDOC_CC* SetObjectAnnotation)(void* device, void* object, const char* key, int valueType, uint32_t valueVectorWidth, const void* value);
+        void (RENDERDOC_CC* SetCommandAnnotation)(void* device, void* queueOrCommandBuffer, const char* key, int valueType, uint32_t valueVectorWidth, const void* value);
+    };
+
+    HMODULE m_renderDocModule = nullptr;
+    RENDERDOC_API_1_7_0* m_api = nullptr;
+    void* m_device = nullptr;
+    void* m_window = nullptr;
+    bool m_isLoaded = false;
+    bool m_initialized = false;
+};
+
+} // namespace Debug
+} // namespace XCEngine

engine/src/Debug/LogCategory.cpp

@@ -14,6 +14,7 @@ const char* LogCategoryToString(LogCategory category) {
         case LogCategory::Memory:    return "Memory";
         case LogCategory::Threading: return "Threading";
         case LogCategory::FileSystem: return "FileSystem";
+        case LogCategory::RenderDoc: return "RenderDoc";
         case LogCategory::Custom:    return "Custom";
         default: return "Unknown";
     }

engine/src/Debug/RenderDocCapture.cpp

@@ -0,0 +1,147 @@
+#include "XCEngine/Debug/RenderDocCapture.h"
+#include "XCEngine/Debug/Logger.h"
+#include <cstdio>
+#include <cstring>
+#include <windows.h>
+
+namespace XCEngine {
+namespace Debug {
+
+static const char* RENDERDOC_DLL_PATH = "renderdoc.dll";
+
+RenderDocCapture& RenderDocCapture::Get() {
+    static RenderDocCapture instance;
+    return instance;
+}
+
+bool RenderDocCapture::Initialize(void* device, void* window) {
+    if (m_initialized) {
+        return true;
+    }
+
+    m_device = device;
+    m_window = window;
+
+    if (!LoadRenderDoc()) {
+        return false;
+    }
+
+    m_api->SetCaptureOptionU32(2, 1);
+    m_api->SetCaptureOptionU32(8, 1);
+    m_api->SetCaptureOptionU32(9, 1);
+    m_api->SetCaptureFilePathTemplate(".\\captures");
+
+    m_initialized = true;
+    Logger::Get().Info(LogCategory::General, "RenderDocCapture initialized successfully");
+    return true;
+}
+
+void RenderDocCapture::Shutdown() {
+    if (m_initialized) {
+        if (IsCapturing()) {
+            EndCapture();
+        }
+        UnloadRenderDoc();
+        m_initialized = false;
+    }
+}
+
+bool RenderDocCapture::LoadRenderDoc() {
+    char exePath[MAX_PATH];
+    GetModuleFileNameA(NULL, exePath, MAX_PATH);
+    char* end = strrchr(exePath, '\\');
+    if (end) *end = '\0';
+    strcat_s(exePath, "\\renderdoc.dll");
+    
+    HMODULE module = LoadLibraryA(exePath);
+    if (!module) {
+        DWORD error = GetLastError();
+        char buf[512];
+        sprintf(buf, "Failed to load renderdoc.dll from %s, error=%lu", exePath, error);
+        Logger::Get().Warning(LogCategory::General, buf);
+        return false;
+    }
+
+    using PFN_RENDERDOC_GetAPI = int (RENDERDOC_CC*)(int version, void** apiOut);
+    PFN_RENDERDOC_GetAPI GetAPI = (PFN_RENDERDOC_GetAPI)GetProcAddress(module, "RENDERDOC_GetAPI");
+    if (!GetAPI) {
+        Logger::Get().Error(LogCategory::General, "Failed to get RENDERDOC_GetAPI");
+        FreeLibrary(module);
+        return false;
+    }
+
+    int apiVersion = 10700;
+    void* apiPtr = nullptr;
+    int ret = GetAPI(apiVersion, &apiPtr);
+    if (ret != 1 || !apiPtr) {
+        char buf[256];
+        sprintf(buf, "Failed to get RenderDoc API: ret=%d, ptr=%p", ret, apiPtr);
+        Logger::Get().Error(LogCategory::General, buf);
+        FreeLibrary(module);
+        return false;
+    }
+
+    m_renderDocModule = module;
+    m_api = (RENDERDOC_API_1_7_0*)apiPtr;
+    m_isLoaded = true;
+
+    return true;
+}
+
+void RenderDocCapture::UnloadRenderDoc() {
+    if (m_renderDocModule) {
+        FreeLibrary(m_renderDocModule);
+        m_renderDocModule = nullptr;
+    }
+    m_api = nullptr;
+    m_isLoaded = false;
+}
+
+bool RenderDocCapture::IsCapturing() const {
+    if (!m_isLoaded || !m_api) {
+        return false;
+    }
+    return m_api->IsFrameCapturing() != 0;
+}
+
+void RenderDocCapture::BeginCapture(const char* title) {
+    if (!m_isLoaded || !m_api) {
+        return;
+    }
+    if (title) {
+        m_api->SetCaptureTitle(title);
+    }
+    m_api->SetActiveWindow(m_device, m_window);
+    m_api->StartFrameCapture(m_device, m_window);
+}
+
+void RenderDocCapture::EndCapture() {
+    if (!m_isLoaded || !m_api) {
+        return;
+    }
+    m_api->EndFrameCapture(m_device, m_window);
+}
+
+void RenderDocCapture::TriggerCapture() {
+    if (!m_isLoaded || !m_api) {
+        return;
+    }
+    m_api->TriggerCapture();
+}
+
+void RenderDocCapture::SetCaptureFilePath(const char* path) {
+    if (!m_isLoaded || !m_api) {
+        return;
+    }
+    m_api->SetCaptureFilePathTemplate(path);
+}
+
+void RenderDocCapture::SetCaptureComments(const char* comments) {
+    if (!m_isLoaded || !m_api) {
+        return;
+    }
+    m_api->SetCaptureFileComments(nullptr, comments);
+}
+
+} // namespace Debug
+} // namespace XCEngine

tests/debug/CMakeLists.txt

@@ -3,6 +3,7 @@ find_package(GTest REQUIRED)
 add_executable(debug_tests
     test_logger.cpp
     test_profiler.cpp
+    test_renderdoc_capture.cpp
 )
 
 target_link_libraries(debug_tests
@@ -12,5 +13,11 @@ target_link_libraries(debug_tests
     GTest::gtest_main
 )
 
+add_custom_command(TARGET debug_tests POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+        ${CMAKE_SOURCE_DIR}/engine/third_party/renderdoc/renderdoc.dll
+        $<TARGET_FILE_DIR:debug_tests>/
+)
+
 include(GoogleTest)
 gtest_discover_tests(debug_tests)

tests/debug/test_renderdoc_capture.cpp

@@ -0,0 +1,51 @@
+#include <gtest/gtest.h>
+#include "XCEngine/Debug/RenderDocCapture.h"
+
+using namespace XCEngine::Debug;
+
+TEST(RenderDocCaptureTest, Initialize) {
+    auto* rc = &RenderDocCapture::Get();
+    bool result = rc->Initialize(nullptr, nullptr);
+    EXPECT_TRUE(result) << "RenderDoc initialization failed - check if renderdoc.dll is available";
+    EXPECT_TRUE(rc->IsLoaded());
+}
+
+TEST(RenderDocCaptureTest, Shutdown) {
+    auto* rc = &RenderDocCapture::Get();
+    rc->Initialize(nullptr, nullptr);
+    ASSERT_TRUE(rc->IsLoaded()) << "RenderDoc not loaded";
+    rc->Shutdown();
+}
+
+TEST(RenderDocCaptureTest, BeginEndCapture) {
+    auto* rc = &RenderDocCapture::Get();
+    rc->Initialize(nullptr, nullptr);
+    ASSERT_TRUE(rc->IsLoaded()) << "RenderDoc not loaded";
+
+    rc->BeginCapture("UnitTestCapture");
+    rc->EndCapture();
+}
+
+TEST(RenderDocCaptureTest, TriggerCapture) {
+    auto* rc = &RenderDocCapture::Get();
+    rc->Initialize(nullptr, nullptr);
+    ASSERT_TRUE(rc->IsLoaded()) << "RenderDoc not loaded";
+
+    rc->TriggerCapture();
+}
+
+TEST(RenderDocCaptureTest, SetCaptureFilePath) {
+    auto* rc = &RenderDocCapture::Get();
+    rc->Initialize(nullptr, nullptr);
+    ASSERT_TRUE(rc->IsLoaded()) << "RenderDoc not loaded";
+
+    rc->SetCaptureFilePath(".\\test_captures");
+}
+
+TEST(RenderDocCaptureTest, SetCaptureComments) {
+    auto* rc = &RenderDocCapture::Get();
+    rc->Initialize(nullptr, nullptr);
+    ASSERT_TRUE(rc->IsLoaded()) << "RenderDoc not loaded";
+
+    rc->SetCaptureComments("Unit test capture");
+}