refactor: extract viewport render flow helpers

editor/src/Viewport/ViewportHostRenderFlowUtils.h

@@ -0,0 +1,148 @@
+#pragma once
+
+#include "ViewportHostRenderTargets.h"
+
+#include <XCEngine/Core/Math/Color.h>
+#include <XCEngine/Rendering/CameraRenderRequest.h>
+
+#include <string>
+
+namespace XCEngine {
+namespace Editor {
+
+struct ViewportRenderFallbackPolicy {
+    const char* statusText = nullptr;
+    Math::Color clearColor = Math::Color::Black();
+    bool shouldClear = false;
+    bool setStatusIfEmpty = false;
+    bool invalidateObjectIdFrame = true;
+};
+
+enum class SceneViewportRenderFailure {
+    MissingSceneViewCamera,
+    NoActiveScene,
+    SceneRendererFailed
+};
+
+enum class GameViewportRenderFailure {
+    NoActiveScene,
+    NoCameraInScene,
+    SceneRendererFailed
+};
+
+inline void SetViewportStatusIfEmpty(std::string& statusText, const char* message) {
+    if (statusText.empty()) {
+        statusText = message;
+    }
+}
+
+inline void ApplyViewportFailureStatus(
+    std::string& statusText,
+    const ViewportRenderFallbackPolicy& policy) {
+    if (policy.statusText == nullptr) {
+        return;
+    }
+
+    if (policy.setStatusIfEmpty) {
+        SetViewportStatusIfEmpty(statusText, policy.statusText);
+        return;
+    }
+
+    statusText = policy.statusText;
+}
+
+inline void InvalidateViewportObjectIdFrame(ViewportRenderTargets& targets) {
+    targets.hasValidObjectIdFrame = false;
+}
+
+inline ViewportRenderFallbackPolicy BuildViewportRenderTargetUnavailablePolicy() {
+    ViewportRenderFallbackPolicy policy = {};
+    policy.statusText = "Viewport render target is unavailable";
+    return policy;
+}
+
+inline ViewportRenderFallbackPolicy BuildSceneViewportRenderFailurePolicy(
+    SceneViewportRenderFailure failure) {
+    ViewportRenderFallbackPolicy policy = {};
+    policy.shouldClear = true;
+
+    switch (failure) {
+    case SceneViewportRenderFailure::MissingSceneViewCamera:
+        policy.statusText = "Scene view camera is unavailable";
+        policy.clearColor = Math::Color(0.18f, 0.07f, 0.07f, 1.0f);
+        break;
+    case SceneViewportRenderFailure::NoActiveScene:
+        policy.statusText = "No active scene";
+        policy.clearColor = Math::Color(0.07f, 0.08f, 0.10f, 1.0f);
+        break;
+    case SceneViewportRenderFailure::SceneRendererFailed:
+        policy.statusText = "Scene renderer failed";
+        policy.clearColor = Math::Color(0.18f, 0.07f, 0.07f, 1.0f);
+        policy.setStatusIfEmpty = true;
+        break;
+    default:
+        break;
+    }
+
+    return policy;
+}
+
+inline ViewportRenderFallbackPolicy BuildGameViewportRenderFailurePolicy(
+    GameViewportRenderFailure failure) {
+    ViewportRenderFallbackPolicy policy = {};
+    policy.shouldClear = true;
+
+    switch (failure) {
+    case GameViewportRenderFailure::NoActiveScene:
+        policy.statusText = "No active scene";
+        policy.clearColor = Math::Color(0.07f, 0.08f, 0.10f, 1.0f);
+        break;
+    case GameViewportRenderFailure::NoCameraInScene:
+        policy.statusText = "No camera in scene";
+        policy.clearColor = Math::Color(0.10f, 0.09f, 0.08f, 1.0f);
+        break;
+    case GameViewportRenderFailure::SceneRendererFailed:
+        policy.statusText = "Scene renderer failed";
+        policy.clearColor = Math::Color(0.18f, 0.07f, 0.07f, 1.0f);
+        break;
+    default:
+        break;
+    }
+
+    return policy;
+}
+
+inline void ApplySceneViewportRenderRequestSetup(
+    const ViewportRenderTargets& targets,
+    Rendering::RenderPassSequence* postPasses,
+    Rendering::CameraRenderRequest& request) {
+    request.postScenePasses = nullptr;
+    request.objectId = {};
+
+    if (postPasses != nullptr && postPasses->GetPassCount() > 0) {
+        request.postScenePasses = postPasses;
+    }
+
+    if (targets.objectIdView == nullptr) {
+        return;
+    }
+
+    request.objectId.surface = BuildViewportObjectIdSurface(targets);
+    request.objectId.surface.SetRenderArea(request.surface.GetRenderArea());
+}
+
+inline void MarkSceneViewportRenderSuccess(
+    ViewportRenderTargets& targets,
+    const Rendering::CameraRenderRequest& request) {
+    targets.colorState = RHI::ResourceStates::PixelShaderResource;
+    targets.objectIdState = RHI::ResourceStates::PixelShaderResource;
+    targets.hasValidObjectIdFrame = request.objectId.IsRequested();
+}
+
+inline void MarkGameViewportRenderSuccess(ViewportRenderTargets& targets) {
+    targets.colorState = RHI::ResourceStates::PixelShaderResource;
+    targets.hasValidObjectIdFrame = false;
+}
+
+} // namespace Editor
+} // namespace XCEngine

editor/src/Viewport/ViewportHostService.h

@@ -6,6 +6,7 @@
 #include "IViewportHostService.h"
 #include "SceneViewportPicker.h"
 #include "SceneViewportCameraController.h"
+#include "ViewportHostRenderFlowUtils.h"
 #include "ViewportHostRenderTargets.h"
 #include "ViewportObjectIdPicker.h"
 #include "UI/ImGuiBackendBridge.h"
@@ -70,12 +71,6 @@ std::unique_ptr<Rendering::RenderPass> MakeLambdaRenderPass(const char* name, Ca
         LambdaRenderPass::ExecuteCallback(std::forward<Callback>(callback)));
 }
 
-inline void SetViewportStatusIfEmpty(std::string& statusText, const char* message) {
-    if (statusText.empty()) {
-        statusText = message;
-    }
-}
-
 Rendering::Passes::InfiniteGridPassData BuildInfiniteGridPassData(
     const SceneViewportOverlayData& overlay) {
     Rendering::Passes::InfiniteGridPassData data = {};
@@ -429,8 +424,26 @@ private:
         return BuildViewportColorSurface(entry.renderTargets);
     }
 
-    Rendering::RenderSurface BuildObjectIdSurface(const ViewportEntry& entry) const {
-        return BuildViewportObjectIdSurface(entry.renderTargets);
+    void ApplyViewportRenderFailure(
+        ViewportEntry& entry,
+        const Rendering::RenderContext& renderContext,
+        const ViewportRenderFallbackPolicy& policy) {
+        ApplyViewportFailureStatus(entry.statusText, policy);
+        if (policy.invalidateObjectIdFrame) {
+            InvalidateViewportObjectIdFrame(entry.renderTargets);
+        }
+
+        if (!policy.shouldClear) {
+            return;
+        }
+
+        ClearViewport(
+            entry,
+            renderContext,
+            policy.clearColor.r,
+            policy.clearColor.g,
+            policy.clearColor.b,
+            policy.clearColor.a);
     }
 
     void AddSceneColorToRenderTargetPass(
@@ -633,18 +646,22 @@ private:
         Rendering::RenderSurface& surface) {
         surface.SetClearColorOverride(Math::Color(0.27f, 0.27f, 0.27f, 1.0f));
         if (!EnsureSceneViewCamera()) {
-            entry.statusText = "Scene view camera is unavailable";
-            ClearViewport(entry, renderContext, 0.18f, 0.07f, 0.07f, 1.0f);
-            entry.renderTargets.hasValidObjectIdFrame = false;
+            ApplyViewportRenderFailure(
+                entry,
+                renderContext,
+                BuildSceneViewportRenderFailurePolicy(
+                    SceneViewportRenderFailure::MissingSceneViewCamera));
             return false;
         }
 
         ApplySceneViewCameraController();
         entry.statusText.clear();
         if (scene == nullptr) {
-            entry.statusText = "No active scene";
-            ClearViewport(entry, renderContext, 0.07f, 0.08f, 0.10f, 1.0f);
-            entry.renderTargets.hasValidObjectIdFrame = false;
+            ApplyViewportRenderFailure(
+                entry,
+                renderContext,
+                BuildSceneViewportRenderFailurePolicy(
+                    SceneViewportRenderFailure::NoActiveScene));
             return false;
         }
 
@@ -654,31 +671,29 @@ private:
         std::vector<Rendering::CameraRenderRequest> requests =
             m_sceneRenderer->BuildRenderRequests(*scene, m_sceneViewCamera.camera, renderContext, surface);
         if (requests.empty()) {
-            SetViewportStatusIfEmpty(entry.statusText, "Scene renderer failed");
-            ClearViewport(entry, renderContext, 0.18f, 0.07f, 0.07f, 1.0f);
-            entry.renderTargets.hasValidObjectIdFrame = false;
+            ApplyViewportRenderFailure(
+                entry,
+                renderContext,
+                BuildSceneViewportRenderFailurePolicy(
+                    SceneViewportRenderFailure::SceneRendererFailed));
             return false;
         }
 
-        if (sceneState.postPasses.GetPassCount() > 0) {
-            requests[0].postScenePasses = &sceneState.postPasses;
-        }
-
-        if (entry.renderTargets.objectIdView != nullptr) {
-            requests[0].objectId.surface = BuildObjectIdSurface(entry);
-            requests[0].objectId.surface.SetRenderArea(requests[0].surface.GetRenderArea());
-        }
+        ApplySceneViewportRenderRequestSetup(
+            entry.renderTargets,
+            &sceneState.postPasses,
+            requests[0]);
 
         if (!m_sceneRenderer->Render(requests)) {
-            SetViewportStatusIfEmpty(entry.statusText, "Scene renderer failed");
-            ClearViewport(entry, renderContext, 0.18f, 0.07f, 0.07f, 1.0f);
-            entry.renderTargets.hasValidObjectIdFrame = false;
+            ApplyViewportRenderFailure(
+                entry,
+                renderContext,
+                BuildSceneViewportRenderFailurePolicy(
+                    SceneViewportRenderFailure::SceneRendererFailed));
             return false;
         }
 
-        entry.renderTargets.colorState = RHI::ResourceStates::PixelShaderResource;
-        entry.renderTargets.objectIdState = RHI::ResourceStates::PixelShaderResource;
-        entry.renderTargets.hasValidObjectIdFrame = requests[0].objectId.IsRequested();
+        MarkSceneViewportRenderSuccess(entry.renderTargets, requests[0]);
         return true;
     }
 
@@ -688,26 +703,34 @@ private:
         const Rendering::RenderContext& renderContext,
         const Rendering::RenderSurface& surface) {
         if (scene == nullptr) {
-            entry.statusText = "No active scene";
-            ClearViewport(entry, renderContext, 0.07f, 0.08f, 0.10f, 1.0f);
+            ApplyViewportRenderFailure(
+                entry,
+                renderContext,
+                BuildGameViewportRenderFailurePolicy(
+                    GameViewportRenderFailure::NoActiveScene));
             return false;
         }
 
         const auto cameras = scene->FindObjectsOfType<Components::CameraComponent>();
         if (cameras.empty()) {
-            entry.statusText = "No camera in scene";
-            ClearViewport(entry, renderContext, 0.10f, 0.09f, 0.08f, 1.0f);
+            ApplyViewportRenderFailure(
+                entry,
+                renderContext,
+                BuildGameViewportRenderFailurePolicy(
+                    GameViewportRenderFailure::NoCameraInScene));
             return false;
         }
 
         if (!m_sceneRenderer->Render(*scene, nullptr, renderContext, surface)) {
-            entry.statusText = "Scene renderer failed";
-            ClearViewport(entry, renderContext, 0.18f, 0.07f, 0.07f, 1.0f);
-            entry.renderTargets.hasValidObjectIdFrame = false;
+            ApplyViewportRenderFailure(
+                entry,
+                renderContext,
+                BuildGameViewportRenderFailurePolicy(
+                    GameViewportRenderFailure::SceneRendererFailed));
             return false;
         }
 
-        entry.renderTargets.colorState = RHI::ResourceStates::PixelShaderResource;
+        MarkGameViewportRenderSuccess(entry.renderTargets);
         entry.statusText.clear();
         return true;
     }
@@ -718,7 +741,10 @@ private:
         const Components::Scene* scene,
         const Rendering::RenderContext& renderContext) {
         if (entry.renderTargets.colorView == nullptr || entry.renderTargets.depthView == nullptr) {
-            entry.statusText = "Viewport render target is unavailable";
+            ApplyViewportFailureStatus(
+                entry.statusText,
+                BuildViewportRenderTargetUnavailablePolicy());
+            InvalidateViewportObjectIdFrame(entry.renderTargets);
             return;
         }
 

tests/editor/CMakeLists.txt

@@ -13,6 +13,7 @@ set(EDITOR_TEST_SOURCES
     test_viewport_host_surface_utils.cpp
     test_viewport_object_id_picker.cpp
     test_viewport_render_targets.cpp
+    test_viewport_render_flow_utils.cpp
     test_builtin_icon_layout_utils.cpp
     ${CMAKE_SOURCE_DIR}/editor/src/Core/UndoManager.cpp
     ${CMAKE_SOURCE_DIR}/editor/src/Managers/SceneManager.cpp

tests/editor/test_viewport_render_flow_utils.cpp

@@ -0,0 +1,223 @@
+#include <gtest/gtest.h>
+
+#include "Viewport/ViewportHostRenderFlowUtils.h"
+
+#include <memory>
+#include <string>
+
+namespace {
+
+using XCEngine::Editor::ApplySceneViewportRenderRequestSetup;
+using XCEngine::Editor::ApplyViewportFailureStatus;
+using XCEngine::Editor::BuildGameViewportRenderFailurePolicy;
+using XCEngine::Editor::BuildSceneViewportRenderFailurePolicy;
+using XCEngine::Editor::BuildViewportRenderTargetUnavailablePolicy;
+using XCEngine::Editor::GameViewportRenderFailure;
+using XCEngine::Editor::MarkGameViewportRenderSuccess;
+using XCEngine::Editor::MarkSceneViewportRenderSuccess;
+using XCEngine::Editor::SceneViewportRenderFailure;
+using XCEngine::Editor::ViewportRenderTargets;
+using XCEngine::RHI::Format;
+using XCEngine::RHI::RHIResourceView;
+using XCEngine::RHI::ResourceStates;
+using XCEngine::RHI::ResourceViewDimension;
+using XCEngine::RHI::ResourceViewType;
+using XCEngine::Rendering::RenderPass;
+using XCEngine::Rendering::RenderPassContext;
+using XCEngine::Rendering::RenderPassSequence;
+using XCEngine::Rendering::RenderSurface;
+
+class DummyResourceView final : public RHIResourceView {
+public:
+    explicit DummyResourceView(
+        ResourceViewType viewType = ResourceViewType::RenderTarget,
+        Format format = Format::R8G8B8A8_UNorm)
+        : m_viewType(viewType)
+        , m_format(format) {
+    }
+
+    void Shutdown() override {
+    }
+
+    void* GetNativeHandle() override {
+        return nullptr;
+    }
+
+    bool IsValid() const override {
+        return true;
+    }
+
+    ResourceViewType GetViewType() const override {
+        return m_viewType;
+    }
+
+    ResourceViewDimension GetDimension() const override {
+        return ResourceViewDimension::Texture2D;
+    }
+
+    Format GetFormat() const override {
+        return m_format;
+    }
+
+private:
+    ResourceViewType m_viewType = ResourceViewType::RenderTarget;
+    Format m_format = Format::R8G8B8A8_UNorm;
+};
+
+class NoopRenderPass final : public RenderPass {
+public:
+    const char* GetName() const override {
+        return "NoopRenderPass";
+    }
+
+    bool Execute(const RenderPassContext&) override {
+        return true;
+    }
+};
+
+TEST(ViewportRenderFlowUtilsTest, BuildFailurePoliciesExposeExpectedStatusAndClearBehavior) {
+    const auto targetUnavailable = BuildViewportRenderTargetUnavailablePolicy();
+    EXPECT_STREQ(targetUnavailable.statusText, "Viewport render target is unavailable");
+    EXPECT_FALSE(targetUnavailable.shouldClear);
+    EXPECT_TRUE(targetUnavailable.invalidateObjectIdFrame);
+
+    const auto missingSceneCamera =
+        BuildSceneViewportRenderFailurePolicy(SceneViewportRenderFailure::MissingSceneViewCamera);
+    EXPECT_STREQ(missingSceneCamera.statusText, "Scene view camera is unavailable");
+    EXPECT_TRUE(missingSceneCamera.shouldClear);
+    EXPECT_FLOAT_EQ(missingSceneCamera.clearColor.r, 0.18f);
+    EXPECT_FLOAT_EQ(missingSceneCamera.clearColor.g, 0.07f);
+    EXPECT_FLOAT_EQ(missingSceneCamera.clearColor.b, 0.07f);
+    EXPECT_FALSE(missingSceneCamera.setStatusIfEmpty);
+
+    const auto sceneRendererFailed =
+        BuildSceneViewportRenderFailurePolicy(SceneViewportRenderFailure::SceneRendererFailed);
+    EXPECT_STREQ(sceneRendererFailed.statusText, "Scene renderer failed");
+    EXPECT_TRUE(sceneRendererFailed.shouldClear);
+    EXPECT_TRUE(sceneRendererFailed.setStatusIfEmpty);
+
+    const auto noGameCamera =
+        BuildGameViewportRenderFailurePolicy(GameViewportRenderFailure::NoCameraInScene);
+    EXPECT_STREQ(noGameCamera.statusText, "No camera in scene");
+    EXPECT_TRUE(noGameCamera.shouldClear);
+    EXPECT_FLOAT_EQ(noGameCamera.clearColor.r, 0.10f);
+    EXPECT_FLOAT_EQ(noGameCamera.clearColor.g, 0.09f);
+    EXPECT_FLOAT_EQ(noGameCamera.clearColor.b, 0.08f);
+}
+
+TEST(ViewportRenderFlowUtilsTest, ApplyViewportFailureStatusRespectsSetIfEmptyBehavior) {
+    std::string statusText;
+    ApplyViewportFailureStatus(
+        statusText,
+        BuildSceneViewportRenderFailurePolicy(SceneViewportRenderFailure::SceneRendererFailed));
+    EXPECT_EQ(statusText, "Scene renderer failed");
+
+    statusText = "Scene object id shader view is unavailable";
+    ApplyViewportFailureStatus(
+        statusText,
+        BuildSceneViewportRenderFailurePolicy(SceneViewportRenderFailure::SceneRendererFailed));
+    EXPECT_EQ(statusText, "Scene object id shader view is unavailable");
+
+    ApplyViewportFailureStatus(
+        statusText,
+        BuildGameViewportRenderFailurePolicy(GameViewportRenderFailure::NoActiveScene));
+    EXPECT_EQ(statusText, "No active scene");
+}
+
+TEST(ViewportRenderFlowUtilsTest, ApplySceneRenderRequestSetupAttachesOptionalPassesAndObjectIdSurface) {
+    DummyResourceView depthView(ResourceViewType::DepthStencil, Format::D24_UNorm_S8_UInt);
+    DummyResourceView objectIdView(ResourceViewType::RenderTarget);
+
+    ViewportRenderTargets targets = {};
+    targets.width = 800;
+    targets.height = 600;
+    targets.depthView = &depthView;
+    targets.objectIdView = &objectIdView;
+    targets.objectIdState = ResourceStates::Common;
+
+    RenderPassSequence postPasses;
+    postPasses.AddPass(std::make_unique<NoopRenderPass>());
+
+    XCEngine::Rendering::CameraRenderRequest request = {};
+    request.surface = RenderSurface(800, 600);
+    request.surface.SetRenderArea(XCEngine::Math::RectInt(64, 32, 320, 240));
+
+    ApplySceneViewportRenderRequestSetup(targets, &postPasses, request);
+
+    EXPECT_EQ(request.postScenePasses, &postPasses);
+    EXPECT_TRUE(request.objectId.IsRequested());
+    ASSERT_EQ(request.objectId.surface.GetColorAttachments().size(), 1u);
+    EXPECT_EQ(request.objectId.surface.GetColorAttachments()[0], &objectIdView);
+    EXPECT_EQ(request.objectId.surface.GetDepthAttachment(), &depthView);
+
+    const auto requestArea = request.surface.GetRenderArea();
+    const auto objectIdArea = request.objectId.surface.GetRenderArea();
+    EXPECT_EQ(objectIdArea.x, requestArea.x);
+    EXPECT_EQ(objectIdArea.y, requestArea.y);
+    EXPECT_EQ(objectIdArea.width, requestArea.width);
+    EXPECT_EQ(objectIdArea.height, requestArea.height);
+}
+
+TEST(ViewportRenderFlowUtilsTest, ApplySceneRenderRequestSetupSkipsUnavailableOptionalAttachments) {
+    ViewportRenderTargets targets = {};
+    targets.width = 800;
+    targets.height = 600;
+
+    RenderPassSequence postPasses;
+
+    XCEngine::Rendering::CameraRenderRequest request = {};
+    request.postScenePasses = reinterpret_cast<RenderPassSequence*>(static_cast<uintptr_t>(0x1));
+    request.objectId.surface = RenderSurface(1, 1);
+    request.objectId.surface.SetColorAttachment(
+        reinterpret_cast<RHIResourceView*>(static_cast<uintptr_t>(0x2)));
+
+    ApplySceneViewportRenderRequestSetup(targets, &postPasses, request);
+
+    EXPECT_EQ(request.postScenePasses, nullptr);
+    EXPECT_FALSE(request.objectId.IsRequested());
+}
+
+TEST(ViewportRenderFlowUtilsTest, MarkSceneRenderSuccessMovesTargetsToShaderResourceState) {
+    DummyResourceView depthView(ResourceViewType::DepthStencil, Format::D24_UNorm_S8_UInt);
+    DummyResourceView objectIdView(ResourceViewType::RenderTarget);
+
+    ViewportRenderTargets targets = {};
+    targets.width = 640;
+    targets.height = 360;
+    targets.depthView = &depthView;
+    targets.objectIdView = &objectIdView;
+    targets.colorState = ResourceStates::Common;
+    targets.objectIdState = ResourceStates::Common;
+
+    XCEngine::Rendering::CameraRenderRequest request = {};
+    request.surface = RenderSurface(640, 360);
+    ApplySceneViewportRenderRequestSetup(targets, nullptr, request);
+
+    MarkSceneViewportRenderSuccess(targets, request);
+    EXPECT_EQ(targets.colorState, ResourceStates::PixelShaderResource);
+    EXPECT_EQ(targets.objectIdState, ResourceStates::PixelShaderResource);
+    EXPECT_TRUE(targets.hasValidObjectIdFrame);
+
+    ViewportRenderTargets noObjectIdTargets = {};
+    noObjectIdTargets.colorState = ResourceStates::Common;
+    noObjectIdTargets.objectIdState = ResourceStates::Common;
+    XCEngine::Rendering::CameraRenderRequest noObjectIdRequest = {};
+
+    MarkSceneViewportRenderSuccess(noObjectIdTargets, noObjectIdRequest);
+    EXPECT_EQ(noObjectIdTargets.colorState, ResourceStates::PixelShaderResource);
+    EXPECT_EQ(noObjectIdTargets.objectIdState, ResourceStates::PixelShaderResource);
+    EXPECT_FALSE(noObjectIdTargets.hasValidObjectIdFrame);
+}
+
+TEST(ViewportRenderFlowUtilsTest, MarkGameRenderSuccessClearsObjectIdFrameAndUpdatesColorState) {
+    ViewportRenderTargets targets = {};
+    targets.colorState = ResourceStates::Common;
+    targets.hasValidObjectIdFrame = true;
+
+    MarkGameViewportRenderSuccess(targets);
+
+    EXPECT_EQ(targets.colorState, ResourceStates::PixelShaderResource);
+    EXPECT_FALSE(targets.hasValidObjectIdFrame);
+}
+
+} // namespace