refactor: formalize scene viewport object-id picking

editor/src/Viewport/ViewportHostService.h

@@ -4,7 +4,6 @@
 #include "Core/ISceneManager.h"
 #include "Core/ISelectionManager.h"
 #include "IViewportHostService.h"
-#include "SceneViewportPicker.h"
 #include "SceneViewportCameraController.h"
 #include "ViewportHostRenderFlowUtils.h"
 #include "ViewportHostRenderTargets.h"
@@ -159,27 +158,20 @@ public:
             return 0;
         }
 
-        const Components::Scene* scene = context.GetSceneManager().GetScene();
-        if (scene == nullptr) {
+        if (context.GetSceneManager().GetScene() == nullptr) {
             return 0;
         }
 
         ViewportEntry& entry = GetEntry(EditorViewportKind::Scene);
-        uint64_t objectIdEntity = 0;
-        if (TryPickSceneViewEntityWithObjectId(
+        const ViewportObjectIdPickResult objectIdPick =
+            PickSceneViewEntityWithObjectId(
                 entry,
                 viewportSize,
-                viewportMousePosition,
-                objectIdEntity)) {
-            return objectIdEntity;
+                viewportMousePosition);
+        if (objectIdPick.status == ViewportObjectIdPickStatus::ReadbackFailed) {
+            SetViewportStatusIfEmpty(entry.statusText, "Scene object id readback failed");
         }
-
-        SceneViewportPickRequest request = {};
-        request.scene = scene;
-        request.overlay = GetSceneViewOverlayData();
-        request.viewportSize = Math::Vector2(viewportSize.x, viewportSize.y);
-        request.viewportPosition = Math::Vector2(viewportMousePosition.x, viewportMousePosition.y);
-        return PickSceneViewportEntity(request).entityId;
+        return objectIdPick.entityId;
     }
 
     void AlignSceneViewToOrientationAxis(SceneViewportOrientationAxis axis) override {
@@ -576,14 +568,12 @@ private:
         targets.hasValidObjectIdFrame = false;
     }
 
-    bool TryPickSceneViewEntityWithObjectId(
+    ViewportObjectIdPickResult PickSceneViewEntityWithObjectId(
         ViewportEntry& entry,
         const ImVec2& viewportSize,
-        const ImVec2& viewportMousePosition,
-        uint64_t& outEntityId) {
+        const ImVec2& viewportMousePosition) {
         if (m_device == nullptr) {
-            outEntityId = 0;
-            return false;
+            return {};
         }
 
         ViewportObjectIdPickContext pickContext = {};
@@ -596,7 +586,7 @@ private:
         pickContext.viewportSize = viewportSize;
         pickContext.viewportMousePosition = viewportMousePosition;
 
-        return TryPickViewportObjectIdEntity(
+        return PickViewportObjectIdEntity(
             pickContext,
             [this](const ViewportObjectIdReadbackRequest& request, std::array<uint8_t, 4>& outRgba) {
                 return m_device != nullptr &&
@@ -607,8 +597,7 @@ private:
                         request.pixelX,
                         request.pixelY,
                         outRgba);
-            },
-            outEntityId);
+            });
     }
 
     UI::ImGuiBackendBridge* m_backend = nullptr;

editor/src/Viewport/ViewportObjectIdPicker.h

@@ -9,6 +9,7 @@
 
 #include <array>
 #include <cstdint>
+#include <utility>
 
 namespace XCEngine {
 namespace Editor {
@@ -32,6 +33,21 @@ struct ViewportObjectIdReadbackRequest {
     uint32_t pixelY = 0;
 };
 
+enum class ViewportObjectIdPickStatus : uint8_t {
+    Unavailable = 0,
+    Success,
+    ReadbackFailed
+};
+
+struct ViewportObjectIdPickResult {
+    ViewportObjectIdPickStatus status = ViewportObjectIdPickStatus::Unavailable;
+    uint64_t entityId = 0;
+
+    bool HasResolvedSample() const {
+        return status == ViewportObjectIdPickStatus::Success;
+    }
+};
+
 inline bool CanPickViewportObjectId(const ViewportObjectIdPickContext& context) {
     return context.commandQueue != nullptr &&
         context.texture != nullptr &&
@@ -67,28 +83,40 @@ inline bool BuildViewportObjectIdReadbackRequest(
 }
 
 template <typename ReadPixelFn>
-bool TryPickViewportObjectIdEntity(
+ViewportObjectIdPickResult PickViewportObjectIdEntity(
     const ViewportObjectIdPickContext& context,
-    ReadPixelFn&& readPixel,
-    uint64_t& outEntityId) {
-    outEntityId = 0;
+    ReadPixelFn&& readPixel) {
+    ViewportObjectIdPickResult result = {};
 
     ViewportObjectIdReadbackRequest request = {};
     if (!BuildViewportObjectIdReadbackRequest(context, request)) {
-        return false;
+        return result;
     }
 
     std::array<uint8_t, 4> rgba = {};
     if (!readPixel(request, rgba)) {
-        return false;
+        result.status = ViewportObjectIdPickStatus::ReadbackFailed;
+        return result;
     }
 
-    outEntityId = static_cast<uint64_t>(Rendering::DecodeObjectIdFromColor(
+    result.status = ViewportObjectIdPickStatus::Success;
+    result.entityId = static_cast<uint64_t>(Rendering::DecodeObjectIdFromColor(
         rgba[0],
         rgba[1],
         rgba[2],
         rgba[3]));
-    return true;
+    return result;
+}
+
+template <typename ReadPixelFn>
+bool TryPickViewportObjectIdEntity(
+    const ViewportObjectIdPickContext& context,
+    ReadPixelFn&& readPixel,
+    uint64_t& outEntityId) {
+    const ViewportObjectIdPickResult result =
+        PickViewportObjectIdEntity(context, std::forward<ReadPixelFn>(readPixel));
+    outEntityId = result.entityId;
+    return result.HasResolvedSample();
 }
 
 } // namespace Editor