Fix scene selection outline mask path

engine/assets/builtin/shaders/selection-outline.shader

@@ -0,0 +1,131 @@
+Shader "Builtin Selection Outline"
+{
+    HLSLINCLUDE
+    // XC_BUILTIN_SELECTION_OUTLINE_D3D12_SHARED
+    cbuffer OutlineConstants
+    {
+        float4 gViewportSizeAndTexelSize;
+        float4 gOutlineColor;
+        float4 gOutlineInfo;
+        float4 gDepthParams;
+    };
+
+    Texture2D gSelectionMaskTexture;
+    Texture2D gDepthTexture;
+
+    struct VSOutput
+    {
+        float4 position : SV_POSITION;
+    };
+
+    VSOutput MainVS(uint vertexId : SV_VertexID)
+    {
+        // XC_BUILTIN_SELECTION_OUTLINE_D3D12_VS
+        static const float2 positions[3] = {
+            float2(-1.0, -1.0),
+            float2(-1.0,  3.0),
+            float2( 3.0, -1.0)
+        };
+
+        VSOutput output;
+        output.position = float4(positions[vertexId], 0.0, 1.0);
+        return output;
+    }
+
+    int2 ClampPixelCoord(int2 pixelCoord)
+    {
+        const int2 maxCoord = int2(
+            max((int)gViewportSizeAndTexelSize.x - 1, 0),
+            max((int)gViewportSizeAndTexelSize.y - 1, 0));
+        return clamp(pixelCoord, int2(0, 0), maxCoord);
+    }
+
+    float LoadSelectionMask(int2 pixelCoord)
+    {
+        return gSelectionMaskTexture.Load(int3(ClampPixelCoord(pixelCoord), 0)).r;
+    }
+
+    float LoadDepth(int2 pixelCoord)
+    {
+        return gDepthTexture.Load(int3(ClampPixelCoord(pixelCoord), 0)).r;
+    }
+
+    bool IsSelectedPixel(float maskValue)
+    {
+        return maskValue > 0.5;
+    }
+
+    float4 MainPS(VSOutput input) : SV_TARGET
+    {
+        // XC_BUILTIN_SELECTION_OUTLINE_D3D12_PS
+        const int2 pixelCoord = int2(input.position.xy);
+        const bool debugSelectionMask = gOutlineInfo.x > 0.5;
+        const bool centerSelected = IsSelectedPixel(LoadSelectionMask(pixelCoord));
+
+        if (debugSelectionMask) {
+            return centerSelected ? float4(1.0, 1.0, 1.0, 1.0) : float4(0.0, 0.0, 0.0, 1.0);
+        }
+
+        if (centerSelected) {
+            discard;
+        }
+
+        const float centerDepth = LoadDepth(pixelCoord);
+        const int outlineWidth = max((int)gOutlineInfo.y, 1);
+        const float depthThreshold = max(gDepthParams.x, 1.0e-6f);
+
+        float outline = 0.0;
+        [loop]
+        for (int y = -2; y <= 2; ++y) {
+            [loop]
+            for (int x = -2; x <= 2; ++x) {
+                if (x == 0 && y == 0) {
+                    continue;
+                }
+
+                const float distancePixels = length(float2((float)x, (float)y));
+                if (distancePixels > outlineWidth) {
+                    continue;
+                }
+
+                const int2 sampleCoord = pixelCoord + int2(x, y);
+                if (!IsSelectedPixel(LoadSelectionMask(sampleCoord))) {
+                    continue;
+                }
+
+                const float selectedDepth = LoadDepth(sampleCoord);
+                if (!(centerDepth > selectedDepth + depthThreshold)) {
+                    continue;
+                }
+
+                const float weight = saturate(
+                    1.0 - ((distancePixels - 1.0) / max((float)outlineWidth, 1.0)));
+                outline = max(outline, weight);
+            }
+        }
+
+        if (outline <= 0.001) {
+            discard;
+        }
+
+        return float4(gOutlineColor.rgb, gOutlineColor.a * outline);
+    }
+    ENDHLSL
+    SubShader
+    {
+        Pass
+        {
+            Name "SelectionOutline"
+            Tags { "LightMode" = "SelectionOutline" }
+            Cull Off
+            ZWrite Off
+            ZTest Always
+            Blend SrcAlpha OneMinusSrcAlpha, One OneMinusSrcAlpha
+            HLSLPROGRAM
+            #pragma target 4.5
+            #pragma vertex MainVS
+            #pragma fragment MainPS
+            ENDHLSL
+        }
+    }
+}