XCEngine/editor/src/Viewport/SceneViewportSelectionOutlinePass.cpp

#include "SceneViewportSelectionOutlinePass.h"

#include <XCEngine/RHI/RHICommandList.h>
#include <XCEngine/RHI/RHIDevice.h>

#include <cstring>

namespace XCEngine {
namespace Editor {

namespace {

constexpr float kOutlineWidthPixels = 2.0f;

const char kSelectionOutlineCompositeHlsl[] = R"(
cbuffer OutlineConstants : register(b0) {
    float4 gViewportSizeAndTexelSize;
    float4 gOutlineColorAndWidth;
};

Texture2D gSelectionMask : register(t0);
SamplerState gMaskSampler : register(s0);

struct VSOutput {
    float4 position : SV_POSITION;
};

VSOutput MainVS(uint vertexId : SV_VertexID) {
    static const float2 positions[3] = {
        float2(-1.0, -1.0),
        float2(-1.0,  3.0),
        float2( 3.0, -1.0)
    };

    VSOutput output;
    output.position = float4(positions[vertexId], 0.0, 1.0);
    return output;
}

float SampleMask(float2 uv) {
    return gSelectionMask.SampleLevel(gMaskSampler, uv, 0).r;
}

float4 MainPS(VSOutput input) : SV_TARGET {
    const float2 viewportSize = max(gViewportSizeAndTexelSize.xy, float2(1.0, 1.0));
    const float2 texelSize = max(gViewportSizeAndTexelSize.zw, float2(1e-6, 1e-6));
    const float outlineWidth = max(gOutlineColorAndWidth.w, 1.0);

    const float2 uv = input.position.xy / viewportSize;
    const float centerMask = SampleMask(uv);
    if (centerMask > 0.001) {
        discard;
    }

    float outline = 0.0;
    [unroll]
    for (int y = -2; y <= 2; ++y) {
        [unroll]
        for (int x = -2; x <= 2; ++x) {
            if (x == 0 && y == 0) {
                continue;
            }

            const float distancePixels = length(float2((float)x, (float)y));
            if (distancePixels > outlineWidth) {
                continue;
            }

            const float sampleMask = SampleMask(uv + float2((float)x, (float)y) * texelSize);
            const float weight = saturate(1.0 - ((distancePixels - 1.0) / max(outlineWidth, 1.0)));
            outline = max(outline, sampleMask * weight);
        }
    }

    if (outline <= 0.001) {
        discard;
    }

    return float4(gOutlineColorAndWidth.rgb, gOutlineColorAndWidth.a * outline);
}
)";

} // namespace

void SceneViewportSelectionOutlinePass::Shutdown() {
    DestroyResources();
}

bool SceneViewportSelectionOutlinePass::Render(
    const Rendering::RenderContext& renderContext,
    const Rendering::RenderSurface& surface,
    RHI::RHIResourceView* maskTextureView) {
    if (!renderContext.IsValid() ||
        renderContext.backendType != RHI::RHIType::D3D12 ||
        maskTextureView == nullptr) {
        return false;
    }

    if (!EnsureInitialized(renderContext)) {
        return false;
    }

    const std::vector<RHI::RHIResourceView*>& colorAttachments = surface.GetColorAttachments();
    if (colorAttachments.empty() || colorAttachments[0] == nullptr) {
        return false;
    }

    OutlineConstants constants = {};
    constants.viewportSizeAndTexelSize = Math::Vector4(
        static_cast<float>(surface.GetWidth()),
        static_cast<float>(surface.GetHeight()),
        surface.GetWidth() > 0 ? 1.0f / static_cast<float>(surface.GetWidth()) : 0.0f,
        surface.GetHeight() > 0 ? 1.0f / static_cast<float>(surface.GetHeight()) : 0.0f);
    constants.outlineColorAndWidth = Math::Vector4(1.0f, 0.4f, 0.0f, kOutlineWidthPixels);
    m_constantSet->WriteConstant(0, &constants, sizeof(constants));
    m_textureSet->Update(0, maskTextureView);

    RHI::RHICommandList* commandList = renderContext.commandList;
    RHI::RHIResourceView* renderTarget = colorAttachments[0];
    commandList->SetRenderTargets(1, &renderTarget, nullptr);

    const RHI::Viewport viewport = {
        0.0f,
        0.0f,
        static_cast<float>(surface.GetWidth()),
        static_cast<float>(surface.GetHeight()),
        0.0f,
        1.0f
    };
    const RHI::Rect scissorRect = {
        0,
        0,
        static_cast<int32_t>(surface.GetWidth()),
        static_cast<int32_t>(surface.GetHeight())
    };

    commandList->SetViewport(viewport);
    commandList->SetScissorRect(scissorRect);
    commandList->SetPrimitiveTopology(RHI::PrimitiveTopology::TriangleList);
    commandList->SetPipelineState(m_pipelineState);

    RHI::RHIDescriptorSet* descriptorSets[] = { m_constantSet, m_textureSet, m_samplerSet };
    commandList->SetGraphicsDescriptorSets(0, 3, descriptorSets, m_pipelineLayout);
    commandList->Draw(3, 1, 0, 0);
    return true;
}

bool SceneViewportSelectionOutlinePass::EnsureInitialized(const Rendering::RenderContext& renderContext) {
    if (m_pipelineLayout != nullptr &&
        m_pipelineState != nullptr &&
        m_constantPool != nullptr &&
        m_constantSet != nullptr &&
        m_texturePool != nullptr &&
        m_textureSet != nullptr &&
        m_samplerPool != nullptr &&
        m_samplerSet != nullptr &&
        m_sampler != nullptr &&
        m_device == renderContext.device &&
        m_backendType == renderContext.backendType) {
        return true;
    }

    DestroyResources();
    return CreateResources(renderContext);
}

bool SceneViewportSelectionOutlinePass::CreateResources(const Rendering::RenderContext& renderContext) {
    if (!renderContext.IsValid() || renderContext.backendType != RHI::RHIType::D3D12) {
        return false;
    }

    m_device = renderContext.device;
    m_backendType = renderContext.backendType;

    RHI::DescriptorSetLayoutBinding setBindings[3] = {};
    setBindings[0].binding = 0;
    setBindings[0].type = static_cast<uint32_t>(RHI::DescriptorType::CBV);
    setBindings[0].count = 1;
    setBindings[1].binding = 0;
    setBindings[1].type = static_cast<uint32_t>(RHI::DescriptorType::SRV);
    setBindings[1].count = 1;
    setBindings[2].binding = 0;
    setBindings[2].type = static_cast<uint32_t>(RHI::DescriptorType::Sampler);
    setBindings[2].count = 1;

    RHI::DescriptorSetLayoutDesc constantLayout = {};
    constantLayout.bindings = &setBindings[0];
    constantLayout.bindingCount = 1;

    RHI::DescriptorSetLayoutDesc textureLayout = {};
    textureLayout.bindings = &setBindings[1];
    textureLayout.bindingCount = 1;

    RHI::DescriptorSetLayoutDesc samplerLayout = {};
    samplerLayout.bindings = &setBindings[2];
    samplerLayout.bindingCount = 1;

    RHI::DescriptorSetLayoutDesc setLayouts[3] = {};
    setLayouts[0] = constantLayout;
    setLayouts[1] = textureLayout;
    setLayouts[2] = samplerLayout;

    RHI::RHIPipelineLayoutDesc pipelineLayoutDesc = {};
    pipelineLayoutDesc.setLayouts = setLayouts;
    pipelineLayoutDesc.setLayoutCount = 3;
    m_pipelineLayout = m_device->CreatePipelineLayout(pipelineLayoutDesc);
    if (m_pipelineLayout == nullptr) {
        DestroyResources();
        return false;
    }

    RHI::DescriptorPoolDesc constantPoolDesc = {};
    constantPoolDesc.type = RHI::DescriptorHeapType::CBV_SRV_UAV;
    constantPoolDesc.descriptorCount = 1;
    constantPoolDesc.shaderVisible = false;
    m_constantPool = m_device->CreateDescriptorPool(constantPoolDesc);
    if (m_constantPool == nullptr) {
        DestroyResources();
        return false;
    }

    m_constantSet = m_constantPool->AllocateSet(constantLayout);
    if (m_constantSet == nullptr) {
        DestroyResources();
        return false;
    }

    RHI::DescriptorPoolDesc texturePoolDesc = {};
    texturePoolDesc.type = RHI::DescriptorHeapType::CBV_SRV_UAV;
    texturePoolDesc.descriptorCount = 1;
    texturePoolDesc.shaderVisible = true;
    m_texturePool = m_device->CreateDescriptorPool(texturePoolDesc);
    if (m_texturePool == nullptr) {
        DestroyResources();
        return false;
    }

    m_textureSet = m_texturePool->AllocateSet(textureLayout);
    if (m_textureSet == nullptr) {
        DestroyResources();
        return false;
    }

    RHI::DescriptorPoolDesc samplerPoolDesc = {};
    samplerPoolDesc.type = RHI::DescriptorHeapType::Sampler;
    samplerPoolDesc.descriptorCount = 1;
    samplerPoolDesc.shaderVisible = true;
    m_samplerPool = m_device->CreateDescriptorPool(samplerPoolDesc);
    if (m_samplerPool == nullptr) {
        DestroyResources();
        return false;
    }

    m_samplerSet = m_samplerPool->AllocateSet(samplerLayout);
    if (m_samplerSet == nullptr) {
        DestroyResources();
        return false;
    }

    RHI::SamplerDesc samplerDesc = {};
    samplerDesc.filter = static_cast<uint32_t>(RHI::FilterMode::Linear);
    samplerDesc.addressU = static_cast<uint32_t>(RHI::TextureAddressMode::Clamp);
    samplerDesc.addressV = static_cast<uint32_t>(RHI::TextureAddressMode::Clamp);
    samplerDesc.addressW = static_cast<uint32_t>(RHI::TextureAddressMode::Clamp);
    samplerDesc.mipLodBias = 0.0f;
    samplerDesc.maxAnisotropy = 1;
    samplerDesc.comparisonFunc = static_cast<uint32_t>(RHI::ComparisonFunc::Always);
    samplerDesc.minLod = 0.0f;
    samplerDesc.maxLod = 1.0f;
    m_sampler = m_device->CreateSampler(samplerDesc);
    if (m_sampler == nullptr) {
        DestroyResources();
        return false;
    }
    m_samplerSet->UpdateSampler(0, m_sampler);

    RHI::GraphicsPipelineDesc pipelineDesc = {};
    pipelineDesc.pipelineLayout = m_pipelineLayout;
    pipelineDesc.topologyType = static_cast<uint32_t>(RHI::PrimitiveTopologyType::Triangle);
    pipelineDesc.renderTargetCount = 1;
    pipelineDesc.renderTargetFormats[0] = static_cast<uint32_t>(RHI::Format::R8G8B8A8_UNorm);
    pipelineDesc.depthStencilFormat = static_cast<uint32_t>(RHI::Format::Unknown);
    pipelineDesc.sampleCount = 1;

    pipelineDesc.rasterizerState.fillMode = static_cast<uint32_t>(RHI::FillMode::Solid);
    pipelineDesc.rasterizerState.cullMode = static_cast<uint32_t>(RHI::CullMode::None);
    pipelineDesc.rasterizerState.frontFace = static_cast<uint32_t>(RHI::FrontFace::CounterClockwise);
    pipelineDesc.rasterizerState.depthClipEnable = true;

    pipelineDesc.blendState.blendEnable = true;
    pipelineDesc.blendState.srcBlend = static_cast<uint32_t>(RHI::BlendFactor::SrcAlpha);
    pipelineDesc.blendState.dstBlend = static_cast<uint32_t>(RHI::BlendFactor::InvSrcAlpha);
    pipelineDesc.blendState.srcBlendAlpha = static_cast<uint32_t>(RHI::BlendFactor::One);
    pipelineDesc.blendState.dstBlendAlpha = static_cast<uint32_t>(RHI::BlendFactor::InvSrcAlpha);
    pipelineDesc.blendState.blendOp = static_cast<uint32_t>(RHI::BlendOp::Add);
    pipelineDesc.blendState.blendOpAlpha = static_cast<uint32_t>(RHI::BlendOp::Add);
    pipelineDesc.blendState.colorWriteMask = static_cast<uint8_t>(RHI::ColorWriteMask::All);

    pipelineDesc.depthStencilState.depthTestEnable = false;
    pipelineDesc.depthStencilState.depthWriteEnable = false;
    pipelineDesc.depthStencilState.depthFunc = static_cast<uint32_t>(RHI::ComparisonFunc::Always);

    pipelineDesc.vertexShader.source.assign(
        kSelectionOutlineCompositeHlsl,
        kSelectionOutlineCompositeHlsl + std::strlen(kSelectionOutlineCompositeHlsl));
    pipelineDesc.vertexShader.sourceLanguage = RHI::ShaderLanguage::HLSL;
    pipelineDesc.vertexShader.entryPoint = L"MainVS";
    pipelineDesc.vertexShader.profile = L"vs_5_0";

    pipelineDesc.fragmentShader.source.assign(
        kSelectionOutlineCompositeHlsl,
        kSelectionOutlineCompositeHlsl + std::strlen(kSelectionOutlineCompositeHlsl));
    pipelineDesc.fragmentShader.sourceLanguage = RHI::ShaderLanguage::HLSL;
    pipelineDesc.fragmentShader.entryPoint = L"MainPS";
    pipelineDesc.fragmentShader.profile = L"ps_5_0";

    m_pipelineState = m_device->CreatePipelineState(pipelineDesc);
    if (m_pipelineState == nullptr || !m_pipelineState->IsValid()) {
        DestroyResources();
        return false;
    }

    return true;
}

void SceneViewportSelectionOutlinePass::DestroyResources() {
    if (m_pipelineState != nullptr) {
        m_pipelineState->Shutdown();
        delete m_pipelineState;
        m_pipelineState = nullptr;
    }

    if (m_sampler != nullptr) {
        m_sampler->Shutdown();
        delete m_sampler;
        m_sampler = nullptr;
    }

    if (m_samplerSet != nullptr) {
        m_samplerSet->Shutdown();
        delete m_samplerSet;
        m_samplerSet = nullptr;
    }

    if (m_samplerPool != nullptr) {
        m_samplerPool->Shutdown();
        delete m_samplerPool;
        m_samplerPool = nullptr;
    }

    if (m_textureSet != nullptr) {
        m_textureSet->Shutdown();
        delete m_textureSet;
        m_textureSet = nullptr;
    }

    if (m_texturePool != nullptr) {
        m_texturePool->Shutdown();
        delete m_texturePool;
        m_texturePool = nullptr;
    }

    if (m_constantSet != nullptr) {
        m_constantSet->Shutdown();
        delete m_constantSet;
        m_constantSet = nullptr;
    }

    if (m_constantPool != nullptr) {
        m_constantPool->Shutdown();
        delete m_constantPool;
        m_constantPool = nullptr;
    }

    if (m_pipelineLayout != nullptr) {
        m_pipelineLayout->Shutdown();
        delete m_pipelineLayout;
        m_pipelineLayout = nullptr;
    }

    m_device = nullptr;
    m_backendType = RHI::RHIType::D3D12;
}

} // namespace Editor
} // namespace XCEngine