XCEngine/editor/src/Viewport/SceneViewportMath.h

#pragma once

#include "IViewportHostService.h"

#include <XCEngine/Core/Math/Matrix4.h>
#include <XCEngine/Core/Math/Plane.h>
#include <XCEngine/Core/Math/Vector2.h>
#include <XCEngine/Core/Math/Vector4.h>

namespace XCEngine {
namespace Editor {

struct SceneViewportProjectedPoint {
    Math::Vector2 screenPosition = Math::Vector2::Zero();
    float ndcDepth = 0.0f;
    bool visible = false;
};

inline Math::Matrix4x4 BuildSceneViewportViewMatrix(const SceneViewportOverlayData& overlay) {
    const Math::Vector3 right = overlay.cameraRight.Normalized();
    const Math::Vector3 up = overlay.cameraUp.Normalized();
    const Math::Vector3 forward = overlay.cameraForward.Normalized();

    Math::Matrix4x4 view = Math::Matrix4x4::Identity();
    view.m[0][0] = right.x;
    view.m[0][1] = right.y;
    view.m[0][2] = right.z;
    view.m[0][3] = -Math::Vector3::Dot(right, overlay.cameraPosition);

    view.m[1][0] = up.x;
    view.m[1][1] = up.y;
    view.m[1][2] = up.z;
    view.m[1][3] = -Math::Vector3::Dot(up, overlay.cameraPosition);

    view.m[2][0] = forward.x;
    view.m[2][1] = forward.y;
    view.m[2][2] = forward.z;
    view.m[2][3] = -Math::Vector3::Dot(forward, overlay.cameraPosition);
    return view;
}

inline Math::Matrix4x4 BuildSceneViewportProjectionMatrix(
    const SceneViewportOverlayData& overlay,
    float viewportWidth,
    float viewportHeight) {
    const float aspect = viewportHeight > 0.0f
        ? viewportWidth / viewportHeight
        : 1.0f;
    return Math::Matrix4x4::Perspective(
        overlay.verticalFovDegrees * Math::DEG_TO_RAD,
        aspect,
        overlay.nearClipPlane,
        overlay.farClipPlane);
}

inline Math::Matrix4x4 BuildSceneViewportViewProjectionMatrix(
    const SceneViewportOverlayData& overlay,
    float viewportWidth,
    float viewportHeight) {
    return BuildSceneViewportProjectionMatrix(overlay, viewportWidth, viewportHeight) *
           BuildSceneViewportViewMatrix(overlay);
}

inline SceneViewportProjectedPoint ProjectSceneViewportWorldPoint(
    const SceneViewportOverlayData& overlay,
    float viewportWidth,
    float viewportHeight,
    const Math::Vector3& worldPoint) {
    SceneViewportProjectedPoint result = {};
    if (!overlay.valid || viewportWidth <= 1.0f || viewportHeight <= 1.0f) {
        return result;
    }

    const Math::Vector4 clipPoint =
        BuildSceneViewportViewProjectionMatrix(overlay, viewportWidth, viewportHeight) *
        Math::Vector4(worldPoint, 1.0f);
    if (clipPoint.w <= Math::EPSILON) {
        return result;
    }

    const Math::Vector3 ndcPoint = clipPoint.ToVector3() / clipPoint.w;
    result.screenPosition.x = (ndcPoint.x * 0.5f + 0.5f) * viewportWidth;
    result.screenPosition.y = (1.0f - (ndcPoint.y * 0.5f + 0.5f)) * viewportHeight;
    result.ndcDepth = ndcPoint.z;
    result.visible =
        ndcPoint.x >= -1.0f && ndcPoint.x <= 1.0f &&
        ndcPoint.y >= -1.0f && ndcPoint.y <= 1.0f &&
        ndcPoint.z >= 0.0f && ndcPoint.z <= 1.0f;
    return result;
}

inline bool ProjectSceneViewportAxisDirection(
    const SceneViewportOverlayData& overlay,
    const Math::Vector3& worldAxis,
    Math::Vector2& outScreenDirection) {
    if (!overlay.valid) {
        return false;
    }

    const Math::Vector3 viewAxis = BuildSceneViewportViewMatrix(overlay).MultiplyVector(worldAxis.Normalized());
    const Math::Vector2 screenDirection(viewAxis.x, -viewAxis.y);
    if (screenDirection.SqrMagnitude() <= Math::EPSILON) {
        return false;
    }

    outScreenDirection = screenDirection.Normalized();
    return true;
}

inline bool ProjectSceneViewportAxisDirectionAtPoint(
    const SceneViewportOverlayData& overlay,
    float viewportWidth,
    float viewportHeight,
    const Math::Vector3& worldPoint,
    const Math::Vector3& worldAxis,
    Math::Vector2& outScreenDirection,
    float sampleDistance = 1.0f) {
    const Math::Vector3 axis = worldAxis.Normalized();
    if (!overlay.valid ||
        viewportWidth <= 1.0f ||
        viewportHeight <= 1.0f ||
        axis.SqrMagnitude() <= Math::EPSILON ||
        sampleDistance <= Math::EPSILON) {
        return false;
    }

    const Math::Matrix4x4 viewProjection =
        BuildSceneViewportViewProjectionMatrix(overlay, viewportWidth, viewportHeight);
    const Math::Vector4 startClip = viewProjection * Math::Vector4(worldPoint, 1.0f);
    const Math::Vector4 endClip = viewProjection * Math::Vector4(worldPoint + axis * sampleDistance, 1.0f);
    if (startClip.w <= Math::EPSILON || endClip.w <= Math::EPSILON) {
        return ProjectSceneViewportAxisDirection(overlay, axis, outScreenDirection);
    }

    const Math::Vector3 startNdc = startClip.ToVector3() / startClip.w;
    const Math::Vector3 endNdc = endClip.ToVector3() / endClip.w;
    const Math::Vector2 startScreen(
        (startNdc.x * 0.5f + 0.5f) * viewportWidth,
        (1.0f - (startNdc.y * 0.5f + 0.5f)) * viewportHeight);
    const Math::Vector2 endScreen(
        (endNdc.x * 0.5f + 0.5f) * viewportWidth,
        (1.0f - (endNdc.y * 0.5f + 0.5f)) * viewportHeight);

    const Math::Vector2 screenDirection = endScreen - startScreen;
    if (screenDirection.SqrMagnitude() <= Math::EPSILON) {
        return ProjectSceneViewportAxisDirection(overlay, axis, outScreenDirection);
    }

    outScreenDirection = screenDirection.Normalized();
    return true;
}

inline bool ProjectSceneViewportWorldPointClamped(
    const SceneViewportOverlayData& overlay,
    float viewportWidth,
    float viewportHeight,
    const Math::Vector3& worldPoint,
    float edgePadding,
    SceneViewportProjectedPoint& outProjectedPoint) {
    outProjectedPoint = ProjectSceneViewportWorldPoint(overlay, viewportWidth, viewportHeight, worldPoint);
    if (!overlay.valid || viewportWidth <= 1.0f || viewportHeight <= 1.0f || outProjectedPoint.ndcDepth < 0.0f) {
        return false;
    }

    const float minX = edgePadding;
    const float minY = edgePadding;
    const float maxX = viewportWidth - edgePadding;
    const float maxY = viewportHeight - edgePadding;
    outProjectedPoint.screenPosition.x = std::clamp(outProjectedPoint.screenPosition.x, minX, maxX);
    outProjectedPoint.screenPosition.y = std::clamp(outProjectedPoint.screenPosition.y, minY, maxY);
    return true;
}

inline float DistanceToSegmentSquared(
    const Math::Vector2& point,
    const Math::Vector2& segmentStart,
    const Math::Vector2& segmentEnd,
    float* outSegmentT = nullptr) {
    const Math::Vector2 segment = segmentEnd - segmentStart;
    const float segmentLengthSq = segment.SqrMagnitude();
    if (segmentLengthSq <= Math::EPSILON) {
        if (outSegmentT != nullptr) {
            *outSegmentT = 0.0f;
        }
        return (point - segmentStart).SqrMagnitude();
    }

    const float segmentT = std::clamp(
        Math::Vector2::Dot(point - segmentStart, segment) / segmentLengthSq,
        0.0f,
        1.0f);
    if (outSegmentT != nullptr) {
        *outSegmentT = segmentT;
    }

    const Math::Vector2 closestPoint = segmentStart + segment * segmentT;
    return (point - closestPoint).SqrMagnitude();
}

inline Math::Plane BuildSceneViewportPlaneFromPointNormal(
    const Math::Vector3& point,
    const Math::Vector3& normal) {
    const Math::Vector3 planeNormal = normal.Normalized();
    return Math::Plane(planeNormal, -Math::Vector3::Dot(planeNormal, point));
}

inline bool BuildSceneViewportAxisDragPlaneNormal(
    const SceneViewportOverlayData& overlay,
    const Math::Vector3& worldAxis,
    Math::Vector3& outPlaneNormal) {
    if (!overlay.valid || worldAxis.SqrMagnitude() <= Math::EPSILON) {
        return false;
    }

    const Math::Vector3 axis = worldAxis.Normalized();
    const Math::Vector3 candidates[] = {
        Math::Vector3::ProjectOnPlane(overlay.cameraForward.Normalized(), axis),
        Math::Vector3::ProjectOnPlane(overlay.cameraUp.Normalized(), axis),
        Math::Vector3::ProjectOnPlane(overlay.cameraRight.Normalized(), axis),
        Math::Vector3::ProjectOnPlane(Math::Vector3::Up(), axis),
        Math::Vector3::ProjectOnPlane(Math::Vector3::Right(), axis),
        Math::Vector3::ProjectOnPlane(Math::Vector3::Forward(), axis)
    };

    for (const Math::Vector3& candidate : candidates) {
        if (candidate.SqrMagnitude() <= Math::EPSILON) {
            continue;
        }

        outPlaneNormal = candidate.Normalized();
        return true;
    }

    return false;
}

} // namespace Editor
} // namespace XCEngine