rendering: add vulkan hlsl shader compilation

engine/src/RHI/Vulkan/VulkanShaderCompiler.cpp

@@ -300,6 +300,29 @@ std::wstring ShaderStageExtension(ShaderType type) {
     return std::wstring(L".") + ShaderStageArgument(type);
 }
 
+std::wstring ShaderSourceExtension(ShaderType type, bool isHlsl) {
+    if (isHlsl) {
+        switch (type) {
+            case ShaderType::Vertex:
+                return L".vert.hlsl";
+            case ShaderType::Fragment:
+                return L".frag.hlsl";
+            case ShaderType::Geometry:
+                return L".geom.hlsl";
+            case ShaderType::TessControl:
+                return L".tesc.hlsl";
+            case ShaderType::TessEvaluation:
+                return L".tese.hlsl";
+            case ShaderType::Compute:
+                return L".comp.hlsl";
+            default:
+                return L".shader.hlsl";
+        }
+    }
+
+    return ShaderStageExtension(type);
+}
+
 bool CreateTemporaryPath(const wchar_t* extension, std::wstring& outPath) {
     wchar_t tempDirectory[MAX_PATH] = {};
     if (GetTempPathW(MAX_PATH, tempDirectory) == 0) {
@@ -427,7 +450,7 @@ bool CompileGlslToSpirv(const ShaderCompileDesc& desc,
 
     std::wstring tempSourcePath;
     std::wstring tempOutputPath;
-    if (!CreateTemporaryPath(ShaderStageExtension(type).c_str(), tempSourcePath) ||
+    if (!CreateTemporaryPath(ShaderSourceExtension(type, false).c_str(), tempSourcePath) ||
         !CreateTemporaryPath(L".spv", tempOutputPath)) {
         if (errorMessage != nullptr) {
             *errorMessage = "Failed to allocate temporary shader compiler files.";
@@ -489,6 +512,84 @@ bool CompileGlslToSpirv(const ShaderCompileDesc& desc,
     return true;
 }
 
+bool CompileHlslToSpirv(const ShaderCompileDesc& desc,
+                        const std::string& sourceText,
+                        ShaderType type,
+                        const std::string& entryPoint,
+                        std::vector<uint32_t>& spirvWords,
+                        std::string* errorMessage) {
+    std::wstring validatorPath;
+    if (!FindGlslangValidator(validatorPath)) {
+        if (errorMessage != nullptr) {
+            *errorMessage = "glslangValidator.exe was not found. Install Vulkan SDK or add it to PATH.";
+        }
+        return false;
+    }
+
+    std::wstring tempSourcePath;
+    std::wstring tempOutputPath;
+    if (!CreateTemporaryPath(ShaderSourceExtension(type, true).c_str(), tempSourcePath) ||
+        !CreateTemporaryPath(L".spv", tempOutputPath)) {
+        if (errorMessage != nullptr) {
+            *errorMessage = "Failed to allocate temporary shader compiler files.";
+        }
+        return false;
+    }
+
+    const std::string sourceWithMacros = InjectMacrosIntoSource(sourceText, desc.macros);
+    const bool wroteSource = WriteTextFile(tempSourcePath, sourceWithMacros);
+    if (!wroteSource) {
+        DeleteFileW(tempSourcePath.c_str());
+        DeleteFileW(tempOutputPath.c_str());
+        if (errorMessage != nullptr) {
+            *errorMessage = "Failed to write temporary HLSL shader file.";
+        }
+        return false;
+    }
+
+    const std::wstring arguments =
+        L"-D -V --target-env vulkan1.0 -S " + ShaderStageArgument(type) +
+        L" -e " + WidenAscii(entryPoint.c_str()) +
+        L" -o \"" + tempOutputPath + L"\" \"" + tempSourcePath + L"\"";
+
+    DWORD exitCode = 0;
+    std::string compilerOutput;
+    const bool ranProcess = RunProcessAndCapture(validatorPath, arguments, exitCode, compilerOutput);
+
+    std::vector<uint8_t> bytes;
+    const bool loadedOutput = ranProcess && exitCode == 0 && LoadBinaryFile(tempOutputPath, bytes);
+
+    DeleteFileW(tempSourcePath.c_str());
+    DeleteFileW(tempOutputPath.c_str());
+
+    if (!ranProcess) {
+        if (errorMessage != nullptr) {
+            *errorMessage = "Failed to launch glslangValidator.exe.";
+        }
+        return false;
+    }
+
+    if (!loadedOutput) {
+        if (errorMessage != nullptr) {
+            *errorMessage = compilerOutput.empty()
+                ? "glslangValidator.exe failed to compile HLSL to SPIR-V."
+                : compilerOutput;
+        }
+        return false;
+    }
+
+    if (bytes.empty() || (bytes.size() % sizeof(uint32_t)) != 0) {
+        if (errorMessage != nullptr) {
+            *errorMessage = "Compiled SPIR-V payload size is invalid.";
+        }
+        return false;
+    }
+
+    spirvWords.resize(bytes.size() / sizeof(uint32_t));
+    std::memcpy(spirvWords.data(), bytes.data(), bytes.size());
+    return true;
+}
+
 } // namespace
 
 bool CompileVulkanShader(const ShaderCompileDesc& desc,
@@ -554,26 +655,23 @@ bool CompileVulkanShader(const ShaderCompileDesc& desc,
         return true;
     }
 
-    if (IsHlslInput(desc)) {
-        if (errorMessage != nullptr) {
-            *errorMessage = "The Vulkan backend currently supports GLSL or SPIR-V inputs, not HLSL source.";
-        }
-        return false;
-    }
-
     std::string sourceText;
     if (!desc.source.empty()) {
         sourceText.assign(reinterpret_cast<const char*>(desc.source.data()), desc.source.size());
     } else if (!LoadTextFile(std::filesystem::path(desc.fileName), sourceText)) {
         if (errorMessage != nullptr) {
-            *errorMessage = "Failed to read GLSL shader file.";
+            *errorMessage = IsHlslInput(desc)
+                ? "Failed to read HLSL shader file."
+                : "Failed to read GLSL shader file.";
         }
         return false;
     }
 
     if (sourceText.empty()) {
         if (errorMessage != nullptr) {
-            *errorMessage = "GLSL shader source is empty.";
+            *errorMessage = IsHlslInput(desc)
+                ? "HLSL shader source is empty."
+                : "GLSL shader source is empty.";
         }
         return false;
     }
@@ -585,13 +683,24 @@ bool CompileVulkanShader(const ShaderCompileDesc& desc,
         return false;
     }
 
-    if (!CompileGlslToSpirv(desc,
-                            sourceText,
-                            outShader.type,
-                            outShader.entryPoint,
-                            outShader.spirvWords,
-                            errorMessage)) {
-        return false;
+    if (IsHlslInput(desc)) {
+        if (!CompileHlslToSpirv(desc,
+                                sourceText,
+                                outShader.type,
+                                outShader.entryPoint,
+                                outShader.spirvWords,
+                                errorMessage)) {
+            return false;
+        }
+    } else {
+        if (!CompileGlslToSpirv(desc,
+                                sourceText,
+                                outShader.type,
+                                outShader.entryPoint,
+                                outShader.spirvWords,
+                                errorMessage)) {
+            return false;
+        }
     }
 
     ShaderType parsedType = outShader.type;