Fix scene selection outline mask path

engine/assets/builtin/shaders/selection-mask.shader

@@ -0,0 +1,78 @@
+Shader "Builtin Selection Mask"
+{
+    Properties
+    {
+        _BaseColor ("Base Color", Color) = (1,1,1,1) [Semantic(BaseColor)]
+        _Cutoff ("Alpha Cutoff", Range) = 0.5 [Semantic(AlphaCutoff)]
+        _MainTex ("Base Map", 2D) = "white" [Semantic(BaseColorTexture)]
+    }
+    HLSLINCLUDE
+    // XC_BUILTIN_SELECTION_MASK_D3D12_SHARED
+    cbuffer PerObjectConstants
+    {
+        float4x4 gProjectionMatrix;
+        float4x4 gViewMatrix;
+        float4x4 gModelMatrix;
+    };
+
+    cbuffer MaterialConstants
+    {
+        float4 gBaseColorFactor;
+        float4 gAlphaCutoffParams;
+    };
+
+    Texture2D BaseColorTexture;
+    SamplerState LinearClampSampler;
+
+    struct VSInput
+    {
+        float3 position : POSITION;
+        float3 normal : NORMAL;
+        float2 texcoord : TEXCOORD0;
+    };
+
+    struct PSInput
+    {
+        float4 position : SV_POSITION;
+        float2 texcoord : TEXCOORD0;
+    };
+
+    PSInput MainVS(VSInput input)
+    {
+        PSInput output;
+        const float4 positionWS = mul(gModelMatrix, float4(input.position, 1.0f));
+        const float4 positionVS = mul(gViewMatrix, positionWS);
+        output.position = mul(gProjectionMatrix, positionVS);
+        output.texcoord = input.texcoord;
+        return output;
+    }
+
+    float4 MainPS(PSInput input) : SV_TARGET
+    {
+    // XC_BUILTIN_SELECTION_MASK_D3D12_PS
+    #ifdef XC_ALPHA_TEST
+        const float4 baseColor =
+            BaseColorTexture.Sample(LinearClampSampler, input.texcoord) * gBaseColorFactor;
+        clip(baseColor.a - gAlphaCutoffParams.x);
+    #endif
+        return float4(1.0, 1.0, 1.0, 1.0);
+    }
+    ENDHLSL
+    SubShader
+    {
+        Pass
+        {
+            Name "SelectionMask"
+            Tags { "LightMode" = "SelectionMask" }
+            Cull Back
+            ZWrite Off
+            ZTest LEqual
+            HLSLPROGRAM
+            #pragma target 4.5
+            #pragma vertex MainVS
+            #pragma fragment MainPS
+            #pragma shader_feature_local _ XC_ALPHA_TEST
+            ENDHLSL
+        }
+    }
+}

engine/assets/builtin/shaders/selection-outline.shader

@@ -0,0 +1,131 @@
+Shader "Builtin Selection Outline"
+{
+    HLSLINCLUDE
+    // XC_BUILTIN_SELECTION_OUTLINE_D3D12_SHARED
+    cbuffer OutlineConstants
+    {
+        float4 gViewportSizeAndTexelSize;
+        float4 gOutlineColor;
+        float4 gOutlineInfo;
+        float4 gDepthParams;
+    };
+
+    Texture2D gSelectionMaskTexture;
+    Texture2D gDepthTexture;
+
+    struct VSOutput
+    {
+        float4 position : SV_POSITION;
+    };
+
+    VSOutput MainVS(uint vertexId : SV_VertexID)
+    {
+        // XC_BUILTIN_SELECTION_OUTLINE_D3D12_VS
+        static const float2 positions[3] = {
+            float2(-1.0, -1.0),
+            float2(-1.0,  3.0),
+            float2( 3.0, -1.0)
+        };
+
+        VSOutput output;
+        output.position = float4(positions[vertexId], 0.0, 1.0);
+        return output;
+    }
+
+    int2 ClampPixelCoord(int2 pixelCoord)
+    {
+        const int2 maxCoord = int2(
+            max((int)gViewportSizeAndTexelSize.x - 1, 0),
+            max((int)gViewportSizeAndTexelSize.y - 1, 0));
+        return clamp(pixelCoord, int2(0, 0), maxCoord);
+    }
+
+    float LoadSelectionMask(int2 pixelCoord)
+    {
+        return gSelectionMaskTexture.Load(int3(ClampPixelCoord(pixelCoord), 0)).r;
+    }
+
+    float LoadDepth(int2 pixelCoord)
+    {
+        return gDepthTexture.Load(int3(ClampPixelCoord(pixelCoord), 0)).r;
+    }
+
+    bool IsSelectedPixel(float maskValue)
+    {
+        return maskValue > 0.5;
+    }
+
+    float4 MainPS(VSOutput input) : SV_TARGET
+    {
+        // XC_BUILTIN_SELECTION_OUTLINE_D3D12_PS
+        const int2 pixelCoord = int2(input.position.xy);
+        const bool debugSelectionMask = gOutlineInfo.x > 0.5;
+        const bool centerSelected = IsSelectedPixel(LoadSelectionMask(pixelCoord));
+
+        if (debugSelectionMask) {
+            return centerSelected ? float4(1.0, 1.0, 1.0, 1.0) : float4(0.0, 0.0, 0.0, 1.0);
+        }
+
+        if (centerSelected) {
+            discard;
+        }
+
+        const float centerDepth = LoadDepth(pixelCoord);
+        const int outlineWidth = max((int)gOutlineInfo.y, 1);
+        const float depthThreshold = max(gDepthParams.x, 1.0e-6f);
+
+        float outline = 0.0;
+        [loop]
+        for (int y = -2; y <= 2; ++y) {
+            [loop]
+            for (int x = -2; x <= 2; ++x) {
+                if (x == 0 && y == 0) {
+                    continue;
+                }
+
+                const float distancePixels = length(float2((float)x, (float)y));
+                if (distancePixels > outlineWidth) {
+                    continue;
+                }
+
+                const int2 sampleCoord = pixelCoord + int2(x, y);
+                if (!IsSelectedPixel(LoadSelectionMask(sampleCoord))) {
+                    continue;
+                }
+
+                const float selectedDepth = LoadDepth(sampleCoord);
+                if (!(centerDepth > selectedDepth + depthThreshold)) {
+                    continue;
+                }
+
+                const float weight = saturate(
+                    1.0 - ((distancePixels - 1.0) / max((float)outlineWidth, 1.0)));
+                outline = max(outline, weight);
+            }
+        }
+
+        if (outline <= 0.001) {
+            discard;
+        }
+
+        return float4(gOutlineColor.rgb, gOutlineColor.a * outline);
+    }
+    ENDHLSL
+    SubShader
+    {
+        Pass
+        {
+            Name "SelectionOutline"
+            Tags { "LightMode" = "SelectionOutline" }
+            Cull Off
+            ZWrite Off
+            ZTest Always
+            Blend SrcAlpha OneMinusSrcAlpha, One OneMinusSrcAlpha
+            HLSLPROGRAM
+            #pragma target 4.5
+            #pragma vertex MainVS
+            #pragma fragment MainPS
+            ENDHLSL
+        }
+    }
+}