rendering: add vulkan hlsl shader compilation

engine/src/RHI/Vulkan/VulkanShaderCompiler.cpp

@@ -300,6 +300,29 @@ std::wstring ShaderStageExtension(ShaderType type) {
     return std::wstring(L".") + ShaderStageArgument(type);
 }
 
+std::wstring ShaderSourceExtension(ShaderType type, bool isHlsl) {
+    if (isHlsl) {
+        switch (type) {
+            case ShaderType::Vertex:
+                return L".vert.hlsl";
+            case ShaderType::Fragment:
+                return L".frag.hlsl";
+            case ShaderType::Geometry:
+                return L".geom.hlsl";
+            case ShaderType::TessControl:
+                return L".tesc.hlsl";
+            case ShaderType::TessEvaluation:
+                return L".tese.hlsl";
+            case ShaderType::Compute:
+                return L".comp.hlsl";
+            default:
+                return L".shader.hlsl";
+        }
+    }
+
+    return ShaderStageExtension(type);
+}
+
 bool CreateTemporaryPath(const wchar_t* extension, std::wstring& outPath) {
     wchar_t tempDirectory[MAX_PATH] = {};
     if (GetTempPathW(MAX_PATH, tempDirectory) == 0) {
@@ -427,7 +450,7 @@ bool CompileGlslToSpirv(const ShaderCompileDesc& desc,
 
     std::wstring tempSourcePath;
     std::wstring tempOutputPath;
-    if (!CreateTemporaryPath(ShaderStageExtension(type).c_str(), tempSourcePath) ||
+    if (!CreateTemporaryPath(ShaderSourceExtension(type, false).c_str(), tempSourcePath) ||
         !CreateTemporaryPath(L".spv", tempOutputPath)) {
         if (errorMessage != nullptr) {
             *errorMessage = "Failed to allocate temporary shader compiler files.";
@@ -489,6 +512,84 @@ bool CompileGlslToSpirv(const ShaderCompileDesc& desc,
     return true;
 }
 
+bool CompileHlslToSpirv(const ShaderCompileDesc& desc,
+                        const std::string& sourceText,
+                        ShaderType type,
+                        const std::string& entryPoint,
+                        std::vector<uint32_t>& spirvWords,
+                        std::string* errorMessage) {
+    std::wstring validatorPath;
+    if (!FindGlslangValidator(validatorPath)) {
+        if (errorMessage != nullptr) {
+            *errorMessage = "glslangValidator.exe was not found. Install Vulkan SDK or add it to PATH.";
+        }
+        return false;
+    }
+
+    std::wstring tempSourcePath;
+    std::wstring tempOutputPath;
+    if (!CreateTemporaryPath(ShaderSourceExtension(type, true).c_str(), tempSourcePath) ||
+        !CreateTemporaryPath(L".spv", tempOutputPath)) {
+        if (errorMessage != nullptr) {
+            *errorMessage = "Failed to allocate temporary shader compiler files.";
+        }
+        return false;
+    }
+
+    const std::string sourceWithMacros = InjectMacrosIntoSource(sourceText, desc.macros);
+    const bool wroteSource = WriteTextFile(tempSourcePath, sourceWithMacros);
+    if (!wroteSource) {
+        DeleteFileW(tempSourcePath.c_str());
+        DeleteFileW(tempOutputPath.c_str());
+        if (errorMessage != nullptr) {
+            *errorMessage = "Failed to write temporary HLSL shader file.";
+        }
+        return false;
+    }
+
+    const std::wstring arguments =
+        L"-D -V --target-env vulkan1.0 -S " + ShaderStageArgument(type) +
+        L" -e " + WidenAscii(entryPoint.c_str()) +
+        L" -o \"" + tempOutputPath + L"\" \"" + tempSourcePath + L"\"";
+
+    DWORD exitCode = 0;
+    std::string compilerOutput;
+    const bool ranProcess = RunProcessAndCapture(validatorPath, arguments, exitCode, compilerOutput);
+
+    std::vector<uint8_t> bytes;
+    const bool loadedOutput = ranProcess && exitCode == 0 && LoadBinaryFile(tempOutputPath, bytes);
+
+    DeleteFileW(tempSourcePath.c_str());
+    DeleteFileW(tempOutputPath.c_str());
+
+    if (!ranProcess) {
+        if (errorMessage != nullptr) {
+            *errorMessage = "Failed to launch glslangValidator.exe.";
+        }
+        return false;
+    }
+
+    if (!loadedOutput) {
+        if (errorMessage != nullptr) {
+            *errorMessage = compilerOutput.empty()
+                ? "glslangValidator.exe failed to compile HLSL to SPIR-V."
+                : compilerOutput;
+        }
+        return false;
+    }
+
+    if (bytes.empty() || (bytes.size() % sizeof(uint32_t)) != 0) {
+        if (errorMessage != nullptr) {
+            *errorMessage = "Compiled SPIR-V payload size is invalid.";
+        }
+        return false;
+    }
+
+    spirvWords.resize(bytes.size() / sizeof(uint32_t));
+    std::memcpy(spirvWords.data(), bytes.data(), bytes.size());
+    return true;
+}
+
 } // namespace
 
 bool CompileVulkanShader(const ShaderCompileDesc& desc,
@@ -554,26 +655,23 @@ bool CompileVulkanShader(const ShaderCompileDesc& desc,
         return true;
     }
 
-    if (IsHlslInput(desc)) {
-        if (errorMessage != nullptr) {
-            *errorMessage = "The Vulkan backend currently supports GLSL or SPIR-V inputs, not HLSL source.";
-        }
-        return false;
-    }
-
     std::string sourceText;
     if (!desc.source.empty()) {
         sourceText.assign(reinterpret_cast<const char*>(desc.source.data()), desc.source.size());
     } else if (!LoadTextFile(std::filesystem::path(desc.fileName), sourceText)) {
         if (errorMessage != nullptr) {
-            *errorMessage = "Failed to read GLSL shader file.";
+            *errorMessage = IsHlslInput(desc)
+                ? "Failed to read HLSL shader file."
+                : "Failed to read GLSL shader file.";
         }
         return false;
     }
 
     if (sourceText.empty()) {
         if (errorMessage != nullptr) {
-            *errorMessage = "GLSL shader source is empty.";
+            *errorMessage = IsHlslInput(desc)
+                ? "HLSL shader source is empty."
+                : "GLSL shader source is empty.";
         }
         return false;
     }
@@ -585,13 +683,24 @@ bool CompileVulkanShader(const ShaderCompileDesc& desc,
         return false;
     }
 
-    if (!CompileGlslToSpirv(desc,
+    if (IsHlslInput(desc)) {
-                            sourceText,
+        if (!CompileHlslToSpirv(desc,
-                            outShader.type,
+                                sourceText,
-                            outShader.entryPoint,
+                                outShader.type,
-                            outShader.spirvWords,
+                                outShader.entryPoint,
-                            errorMessage)) {
+                                outShader.spirvWords,
-        return false;
+                                errorMessage)) {
+            return false;
+        }
+    } else {
+        if (!CompileGlslToSpirv(desc,
+                                sourceText,
+                                outShader.type,
+                                outShader.entryPoint,
+                                outShader.spirvWords,
+                                errorMessage)) {
+            return false;
+        }
     }
 
     ShaderType parsedType = outShader.type;

tests/RHI/Vulkan/unit/test_pipeline_state.cpp

@@ -53,6 +53,46 @@ void main() {
     delete pipelineState;
 }
 
+TEST_F(VulkanGraphicsFixture, CreateGraphicsPipelineFromHlslShadersProducesValidPipeline) {
+    static const char* vertexSource = R"(
+float4 MainVS(uint vertexId : SV_VertexID) : SV_POSITION
+{
+    const float x = (vertexId == 1u) ? 0.5f : -0.5f;
+    const float y = (vertexId == 2u) ? -0.5f : 0.5f;
+    return float4(x, y, 0.0f, 1.0f);
+}
+)";
+
+    static const char* fragmentSource = R"(
+float4 MainPS() : SV_TARGET
+{
+    return float4(1.0f, 0.0f, 0.0f, 1.0f);
+}
+)";
+
+    GraphicsPipelineDesc pipelineDesc = {};
+    pipelineDesc.topologyType = static_cast<uint32_t>(PrimitiveTopologyType::Triangle);
+    pipelineDesc.renderTargetFormats[0] = static_cast<uint32_t>(Format::R8G8B8A8_UNorm);
+    pipelineDesc.depthStencilFormat = static_cast<uint32_t>(Format::Unknown);
+
+    pipelineDesc.vertexShader.sourceLanguage = ShaderLanguage::HLSL;
+    pipelineDesc.vertexShader.entryPoint = L"MainVS";
+    pipelineDesc.vertexShader.profile = L"vs_5_0";
+    pipelineDesc.vertexShader.source.assign(vertexSource, vertexSource + std::strlen(vertexSource));
+
+    pipelineDesc.fragmentShader.sourceLanguage = ShaderLanguage::HLSL;
+    pipelineDesc.fragmentShader.entryPoint = L"MainPS";
+    pipelineDesc.fragmentShader.profile = L"ps_5_0";
+    pipelineDesc.fragmentShader.source.assign(fragmentSource, fragmentSource + std::strlen(fragmentSource));
+
+    RHIPipelineState* pipelineState = m_device->CreatePipelineState(pipelineDesc);
+    ASSERT_NE(pipelineState, nullptr);
+    EXPECT_TRUE(pipelineState->IsValid());
+    EXPECT_NE(pipelineState->GetNativeHandle(), nullptr);
+    pipelineState->Shutdown();
+    delete pipelineState;
+}
+
 } // namespace
 
 #endif

tests/RHI/Vulkan/unit/test_shader.cpp

@@ -55,6 +55,54 @@ TEST_F(VulkanGraphicsFixture, CreateShaderFromGlslFileProducesValidVertexShader)
     delete shader;
 }
 
+TEST_F(VulkanGraphicsFixture, CreateShaderFromHlslSourceProducesValidVertexShader) {
+    static const char* vertexSource = R"(
+float4 MainVS(uint vertexId : SV_VertexID) : SV_POSITION
+{
+    const float x = (vertexId == 1u) ? 0.5f : -0.5f;
+    const float y = (vertexId == 2u) ? -0.5f : 0.5f;
+    return float4(x, y, 0.0f, 1.0f);
+}
+)";
+
+    ShaderCompileDesc shaderDesc = {};
+    shaderDesc.sourceLanguage = ShaderLanguage::HLSL;
+    shaderDesc.entryPoint = L"MainVS";
+    shaderDesc.profile = L"vs_5_0";
+    shaderDesc.source.assign(vertexSource, vertexSource + std::strlen(vertexSource));
+
+    RHIShader* shader = m_device->CreateShader(shaderDesc);
+    ASSERT_NE(shader, nullptr);
+    EXPECT_TRUE(shader->IsValid());
+    EXPECT_EQ(shader->GetType(), ShaderType::Vertex);
+    EXPECT_NE(shader->GetNativeHandle(), nullptr);
+    shader->Shutdown();
+    delete shader;
+}
+
+TEST_F(VulkanGraphicsFixture, CreateShaderFromHlslSourceProducesValidFragmentShader) {
+    static const char* fragmentSource = R"(
+float4 MainPS() : SV_TARGET
+{
+    return float4(1.0f, 0.0f, 0.0f, 1.0f);
+}
+)";
+
+    ShaderCompileDesc shaderDesc = {};
+    shaderDesc.sourceLanguage = ShaderLanguage::HLSL;
+    shaderDesc.entryPoint = L"MainPS";
+    shaderDesc.profile = L"ps_5_0";
+    shaderDesc.source.assign(fragmentSource, fragmentSource + std::strlen(fragmentSource));
+
+    RHIShader* shader = m_device->CreateShader(shaderDesc);
+    ASSERT_NE(shader, nullptr);
+    EXPECT_TRUE(shader->IsValid());
+    EXPECT_EQ(shader->GetType(), ShaderType::Fragment);
+    EXPECT_NE(shader->GetNativeHandle(), nullptr);
+    shader->Shutdown();
+    delete shader;
+}
+
 TEST_F(VulkanGraphicsFixture, CreateShaderFromGlslSourceInfersComputeShader) {
     RHIShader* shader = CreateWriteRedComputeShaderFromGlsl();
     ASSERT_NE(shader, nullptr);