XCEngine/engine/src/Rendering/Passes/BuiltinObjectIdOutlinePass.cpp

#include "Rendering/Passes/BuiltinObjectIdOutlinePass.h"

#include "Core/Asset/ResourceManager.h"
#include "Debug/Logger.h"
#include "Rendering/Picking/ObjectIdCodec.h"
#include "Rendering/Detail/ShaderVariantUtils.h"
#include "RHI/RHICommandList.h"
#include "RHI/RHIDevice.h"

#include <algorithm>
#include <utility>

namespace XCEngine {
namespace Rendering {
namespace Passes {

namespace {

const Resources::ShaderPass* FindObjectIdOutlineCompatiblePass(
    const Resources::Shader& shader,
    Resources::ShaderBackend backend) {
    const Resources::ShaderPass* outlinePass = shader.FindPass("ObjectIdOutline");
    if (outlinePass != nullptr &&
        ::XCEngine::Rendering::Detail::ShaderPassHasGraphicsVariants(shader, outlinePass->name, backend)) {
        return outlinePass;
    }

    const Resources::ShaderPass* editorOutlinePass = shader.FindPass("EditorObjectIdOutline");
    if (editorOutlinePass != nullptr &&
        ::XCEngine::Rendering::Detail::ShaderPassHasGraphicsVariants(shader, editorOutlinePass->name, backend)) {
        return editorOutlinePass;
    }

    if (shader.GetPassCount() > 0 &&
        ::XCEngine::Rendering::Detail::ShaderPassHasGraphicsVariants(shader, shader.GetPasses()[0].name, backend)) {
        return &shader.GetPasses()[0];
    }

    return nullptr;
}

RHI::GraphicsPipelineDesc CreatePipelineDesc(
    RHI::RHIType backendType,
    RHI::RHIPipelineLayout* pipelineLayout,
    const Resources::Shader& shader,
    const Containers::String& passName) {
    RHI::GraphicsPipelineDesc pipelineDesc = {};
    pipelineDesc.pipelineLayout = pipelineLayout;
    pipelineDesc.topologyType = static_cast<uint32_t>(RHI::PrimitiveTopologyType::Triangle);
    pipelineDesc.renderTargetCount = 1;
    pipelineDesc.renderTargetFormats[0] = static_cast<uint32_t>(RHI::Format::R8G8B8A8_UNorm);
    pipelineDesc.depthStencilFormat = static_cast<uint32_t>(RHI::Format::Unknown);
    pipelineDesc.sampleCount = 1;

    pipelineDesc.rasterizerState.fillMode = static_cast<uint32_t>(RHI::FillMode::Solid);
    pipelineDesc.rasterizerState.cullMode = static_cast<uint32_t>(RHI::CullMode::None);
    pipelineDesc.rasterizerState.frontFace = static_cast<uint32_t>(RHI::FrontFace::CounterClockwise);
    pipelineDesc.rasterizerState.depthClipEnable = true;

    pipelineDesc.blendState.blendEnable = true;
    pipelineDesc.blendState.srcBlend = static_cast<uint32_t>(RHI::BlendFactor::SrcAlpha);
    pipelineDesc.blendState.dstBlend = static_cast<uint32_t>(RHI::BlendFactor::InvSrcAlpha);
    pipelineDesc.blendState.srcBlendAlpha = static_cast<uint32_t>(RHI::BlendFactor::One);
    pipelineDesc.blendState.dstBlendAlpha = static_cast<uint32_t>(RHI::BlendFactor::InvSrcAlpha);
    pipelineDesc.blendState.blendOp = static_cast<uint32_t>(RHI::BlendOp::Add);
    pipelineDesc.blendState.blendOpAlpha = static_cast<uint32_t>(RHI::BlendOp::Add);
    pipelineDesc.blendState.colorWriteMask = static_cast<uint8_t>(RHI::ColorWriteMask::All);

    pipelineDesc.depthStencilState.depthTestEnable = false;
    pipelineDesc.depthStencilState.depthWriteEnable = false;
    pipelineDesc.depthStencilState.depthFunc = static_cast<uint32_t>(RHI::ComparisonFunc::Always);

    const Resources::ShaderBackend backend = ::XCEngine::Rendering::Detail::ToShaderBackend(backendType);
    if (const Resources::ShaderStageVariant* vertexVariant =
            shader.FindVariant(passName, Resources::ShaderType::Vertex, backend)) {
        ::XCEngine::Rendering::Detail::ApplyShaderStageVariant(*vertexVariant, pipelineDesc.vertexShader);
    }
    if (const Resources::ShaderStageVariant* fragmentVariant =
            shader.FindVariant(passName, Resources::ShaderType::Fragment, backend)) {
        ::XCEngine::Rendering::Detail::ApplyShaderStageVariant(*fragmentVariant, pipelineDesc.fragmentShader);
    }

    return pipelineDesc;
}

} // namespace

BuiltinObjectIdOutlinePass::BuiltinObjectIdOutlinePass(Containers::String shaderPath)
    : m_shaderPath(std::move(shaderPath)) {
    ResetState();
}

void BuiltinObjectIdOutlinePass::SetShaderPath(const Containers::String& shaderPath) {
    if (m_shaderPath == shaderPath) {
        return;
    }

    DestroyResources();
    m_shaderPath = shaderPath;
}

const Containers::String& BuiltinObjectIdOutlinePass::GetShaderPath() const {
    return m_shaderPath;
}

void BuiltinObjectIdOutlinePass::Shutdown() {
    DestroyResources();
}

bool BuiltinObjectIdOutlinePass::Render(
    const RenderContext& renderContext,
    const RenderSurface& surface,
    RHI::RHIResourceView* objectIdTextureView,
    const std::vector<uint64_t>& selectedObjectIds,
    const ObjectIdOutlineStyle& style) {
    if (!renderContext.IsValid() ||
        renderContext.backendType != RHI::RHIType::D3D12 ||
        objectIdTextureView == nullptr ||
        selectedObjectIds.empty()) {
        return false;
    }

    if (!EnsureInitialized(renderContext)) {
        return false;
    }

    const std::vector<RHI::RHIResourceView*>& colorAttachments = surface.GetColorAttachments();
    if (colorAttachments.empty() || colorAttachments[0] == nullptr) {
        return false;
    }

    OutlineConstants constants = {};
    constants.viewportSizeAndTexelSize = Math::Vector4(
        static_cast<float>(surface.GetWidth()),
        static_cast<float>(surface.GetHeight()),
        surface.GetWidth() > 0 ? 1.0f / static_cast<float>(surface.GetWidth()) : 0.0f,
        surface.GetHeight() > 0 ? 1.0f / static_cast<float>(surface.GetHeight()) : 0.0f);
    constants.outlineColor = Math::Vector4(
        style.outlineColor.r,
        style.outlineColor.g,
        style.outlineColor.b,
        style.outlineColor.a);

    const uint32_t selectedCount = (std::min)(
        static_cast<uint32_t>(selectedObjectIds.size()),
        kMaxSelectedObjectCount);
    constants.selectedInfo = Math::Vector4(
        static_cast<float>(selectedCount),
        style.debugSelectionMask ? 1.0f : 0.0f,
        style.outlineWidthPixels,
        0.0f);
    for (uint32_t index = 0; index < selectedCount; ++index) {
        constants.selectedObjectColors[index] =
            EncodeObjectIdToColor(selectedObjectIds[index]);
    }

    m_constantSet->WriteConstant(0, &constants, sizeof(constants));
    m_textureSet->Update(0, objectIdTextureView);

    RHI::RHICommandList* commandList = renderContext.commandList;
    RHI::RHIResourceView* renderTarget = colorAttachments[0];
    commandList->SetRenderTargets(1, &renderTarget, nullptr);

    const RHI::Viewport viewport = {
        0.0f,
        0.0f,
        static_cast<float>(surface.GetWidth()),
        static_cast<float>(surface.GetHeight()),
        0.0f,
        1.0f
    };
    const RHI::Rect scissorRect = {
        0,
        0,
        static_cast<int32_t>(surface.GetWidth()),
        static_cast<int32_t>(surface.GetHeight())
    };

    commandList->SetViewport(viewport);
    commandList->SetScissorRect(scissorRect);
    commandList->SetPrimitiveTopology(RHI::PrimitiveTopology::TriangleList);
    commandList->SetPipelineState(m_pipelineState);

    RHI::RHIDescriptorSet* descriptorSets[] = { m_constantSet, m_textureSet };
    commandList->SetGraphicsDescriptorSets(0, 2, descriptorSets, m_pipelineLayout);
    commandList->Draw(3, 1, 0, 0);
    return true;
}

bool BuiltinObjectIdOutlinePass::EnsureInitialized(const RenderContext& renderContext) {
    if (m_pipelineLayout != nullptr &&
        m_pipelineState != nullptr &&
        m_constantPool != nullptr &&
        m_constantSet != nullptr &&
        m_texturePool != nullptr &&
        m_textureSet != nullptr &&
        m_device == renderContext.device &&
        m_backendType == renderContext.backendType) {
        return true;
    }

    if (HasCreatedResources()) {
        DestroyResources();
    }
    return CreateResources(renderContext);
}

bool BuiltinObjectIdOutlinePass::CreateResources(const RenderContext& renderContext) {
    if (!renderContext.IsValid() || renderContext.backendType != RHI::RHIType::D3D12) {
        return false;
    }

    if (m_shaderPath.Empty()) {
        Debug::Logger::Get().Error(
            Debug::LogCategory::Rendering,
            "BuiltinObjectIdOutlinePass requires an injected shader path before resource creation");
        return false;
    }

    Resources::ResourceHandle<Resources::Shader> shader = Resources::ResourceManager::Get().Load<Resources::Shader>(
        m_shaderPath);
    if (!shader.IsValid()) {
        Debug::Logger::Get().Error(
            Debug::LogCategory::Rendering,
            "BuiltinObjectIdOutlinePass failed to load configured object-id-outline shader resource");
        ResetState();
        return false;
    }

    m_device = renderContext.device;
    m_backendType = renderContext.backendType;
    m_builtinObjectIdOutlineShader.emplace(std::move(shader));

    const Resources::ShaderBackend backend = ::XCEngine::Rendering::Detail::ToShaderBackend(m_backendType);
    const Resources::ShaderPass* outlinePass =
        FindObjectIdOutlineCompatiblePass(*m_builtinObjectIdOutlineShader->Get(), backend);
    if (outlinePass == nullptr) {
        Debug::Logger::Get().Error(
            Debug::LogCategory::Rendering,
            "BuiltinObjectIdOutlinePass could not resolve a valid ObjectIdOutline shader pass");
        DestroyResources();
        return false;
    }

    RHI::DescriptorSetLayoutBinding setBindings[2] = {};
    setBindings[0].binding = 0;
    setBindings[0].type = static_cast<uint32_t>(RHI::DescriptorType::CBV);
    setBindings[0].count = 1;
    setBindings[1].binding = 0;
    setBindings[1].type = static_cast<uint32_t>(RHI::DescriptorType::SRV);
    setBindings[1].count = 1;

    RHI::DescriptorSetLayoutDesc constantLayout = {};
    constantLayout.bindings = &setBindings[0];
    constantLayout.bindingCount = 1;

    RHI::DescriptorSetLayoutDesc textureLayout = {};
    textureLayout.bindings = &setBindings[1];
    textureLayout.bindingCount = 1;

    RHI::DescriptorSetLayoutDesc setLayouts[2] = {};
    setLayouts[0] = constantLayout;
    setLayouts[1] = textureLayout;

    RHI::RHIPipelineLayoutDesc pipelineLayoutDesc = {};
    pipelineLayoutDesc.setLayouts = setLayouts;
    pipelineLayoutDesc.setLayoutCount = 2;
    m_pipelineLayout = m_device->CreatePipelineLayout(pipelineLayoutDesc);
    if (m_pipelineLayout == nullptr) {
        DestroyResources();
        return false;
    }

    RHI::DescriptorPoolDesc constantPoolDesc = {};
    constantPoolDesc.type = RHI::DescriptorHeapType::CBV_SRV_UAV;
    constantPoolDesc.descriptorCount = 1;
    constantPoolDesc.shaderVisible = false;
    m_constantPool = m_device->CreateDescriptorPool(constantPoolDesc);
    if (m_constantPool == nullptr) {
        DestroyResources();
        return false;
    }

    m_constantSet = m_constantPool->AllocateSet(constantLayout);
    if (m_constantSet == nullptr) {
        DestroyResources();
        return false;
    }

    RHI::DescriptorPoolDesc texturePoolDesc = {};
    texturePoolDesc.type = RHI::DescriptorHeapType::CBV_SRV_UAV;
    texturePoolDesc.descriptorCount = 1;
    texturePoolDesc.shaderVisible = true;
    m_texturePool = m_device->CreateDescriptorPool(texturePoolDesc);
    if (m_texturePool == nullptr) {
        DestroyResources();
        return false;
    }

    m_textureSet = m_texturePool->AllocateSet(textureLayout);
    if (m_textureSet == nullptr) {
        DestroyResources();
        return false;
    }

    m_pipelineState = m_device->CreatePipelineState(
        CreatePipelineDesc(
                m_backendType,
                m_pipelineLayout,
                *m_builtinObjectIdOutlineShader->Get(),
                outlinePass->name));
    if (m_pipelineState == nullptr || !m_pipelineState->IsValid()) {
        DestroyResources();
        return false;
    }

    return true;
}

bool BuiltinObjectIdOutlinePass::HasCreatedResources() const {
    return m_device != nullptr ||
        m_pipelineLayout != nullptr ||
        m_pipelineState != nullptr ||
        m_constantPool != nullptr ||
        m_constantSet != nullptr ||
        m_texturePool != nullptr ||
        m_textureSet != nullptr ||
        m_builtinObjectIdOutlineShader.has_value();
}

void BuiltinObjectIdOutlinePass::DestroyResources() {
    if (m_pipelineState != nullptr) {
        m_pipelineState->Shutdown();
        delete m_pipelineState;
        m_pipelineState = nullptr;
    }

    if (m_textureSet != nullptr) {
        m_textureSet->Shutdown();
        delete m_textureSet;
        m_textureSet = nullptr;
    }

    if (m_texturePool != nullptr) {
        m_texturePool->Shutdown();
        delete m_texturePool;
        m_texturePool = nullptr;
    }

    if (m_constantSet != nullptr) {
        m_constantSet->Shutdown();
        delete m_constantSet;
        m_constantSet = nullptr;
    }

    if (m_constantPool != nullptr) {
        m_constantPool->Shutdown();
        delete m_constantPool;
        m_constantPool = nullptr;
    }

    if (m_pipelineLayout != nullptr) {
        m_pipelineLayout->Shutdown();
        delete m_pipelineLayout;
        m_pipelineLayout = nullptr;
    }

    if (m_builtinObjectIdOutlineShader.has_value()) {
        m_builtinObjectIdOutlineShader.reset();
    }

    ResetState();
}

void BuiltinObjectIdOutlinePass::ResetState() {
    m_device = nullptr;
    m_backendType = RHI::RHIType::D3D12;
    m_pipelineLayout = nullptr;
    m_pipelineState = nullptr;
    m_constantPool = nullptr;
    m_constantSet = nullptr;
    m_texturePool = nullptr;
    m_textureSet = nullptr;
    m_builtinObjectIdOutlineShader.reset();
}

} // namespace Passes
} // namespace Rendering
} // namespace XCEngine