From d3377708d2777414521910ba0825c4b504e9b465 Mon Sep 17 00:00:00 2001 From: ssdfasd <2156608475@qq.com> Date: Tue, 7 Apr 2026 11:59:50 +0800 Subject: [PATCH] resources: split shader loader internals --- engine/CMakeLists.txt | 10 + .../Resources/Shader/ShaderArtifactLoader.cpp | 292 ++++ .../Resources/Shader/ShaderArtifactLoader.h | 12 + .../Shader/ShaderAuthoringLoader.cpp | 102 ++ .../Resources/Shader/ShaderAuthoringLoader.h | 33 + .../src/Resources/Shader/ShaderFileUtils.cpp | 98 ++ engine/src/Resources/Shader/ShaderFileUtils.h | 22 + engine/src/Resources/Shader/ShaderLoader.cpp | 1240 +---------------- .../Resources/Shader/ShaderManifestLoader.cpp | 518 +++++++ .../Resources/Shader/ShaderManifestLoader.h | 24 + .../Shader/ShaderRuntimeBuildUtils.cpp | 273 ++++ .../Shader/ShaderRuntimeBuildUtils.h | 30 + 12 files changed, 1426 insertions(+), 1228 deletions(-) create mode 100644 engine/src/Resources/Shader/ShaderArtifactLoader.cpp create mode 100644 engine/src/Resources/Shader/ShaderArtifactLoader.h create mode 100644 engine/src/Resources/Shader/ShaderAuthoringLoader.cpp create mode 100644 engine/src/Resources/Shader/ShaderAuthoringLoader.h create mode 100644 engine/src/Resources/Shader/ShaderFileUtils.cpp create mode 100644 engine/src/Resources/Shader/ShaderFileUtils.h create mode 100644 engine/src/Resources/Shader/ShaderManifestLoader.cpp create mode 100644 engine/src/Resources/Shader/ShaderManifestLoader.h create mode 100644 engine/src/Resources/Shader/ShaderRuntimeBuildUtils.cpp create mode 100644 engine/src/Resources/Shader/ShaderRuntimeBuildUtils.h diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 654ed036..eff5073e 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -388,6 +388,16 @@ add_library(XCEngine STATIC ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/Shader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderLoader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderIR.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderFileUtils.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderFileUtils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderRuntimeBuildUtils.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderRuntimeBuildUtils.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderAuthoringLoader.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderAuthoringLoader.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderManifestLoader.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderManifestLoader.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderArtifactLoader.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderArtifactLoader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderSourceUtils.h ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderSourceUtils.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Resources/Shader/ShaderAuthoringParser.h diff --git a/engine/src/Resources/Shader/ShaderArtifactLoader.cpp b/engine/src/Resources/Shader/ShaderArtifactLoader.cpp new file mode 100644 index 00000000..19c5e3af --- /dev/null +++ b/engine/src/Resources/Shader/ShaderArtifactLoader.cpp @@ -0,0 +1,292 @@ +#include "ShaderArtifactLoader.h" + +#include "ShaderFileUtils.h" +#include "ShaderRuntimeBuildUtils.h" + +#include + +#include +#include + +namespace XCEngine { +namespace Resources { + +namespace { + +template +bool ReadShaderArtifactValue(const Containers::Array& data, size_t& offset, T& outValue) { + if (offset + sizeof(T) > data.Size()) { + return false; + } + + std::memcpy(&outValue, data.Data() + offset, sizeof(T)); + offset += sizeof(T); + return true; +} + +bool ReadShaderArtifactString( + const Containers::Array& data, + size_t& offset, + Containers::String& outValue) { + Core::uint32 length = 0; + if (!ReadShaderArtifactValue(data, offset, length)) { + return false; + } + + if (length == 0) { + outValue.Clear(); + return true; + } + + if (offset + length > data.Size()) { + return false; + } + + outValue = Containers::String( + std::string(reinterpret_cast(data.Data() + offset), length).c_str()); + offset += length; + return true; +} + +} // namespace + +LoadResult LoadShaderArtifact(const Containers::String& path) { + const Containers::Array data = ReadShaderFileData(path); + if (data.Empty()) { + return LoadResult("Failed to read shader artifact: " + path); + } + + size_t offset = 0; + ShaderArtifactFileHeader fileHeader; + if (!ReadShaderArtifactValue(data, offset, fileHeader)) { + return LoadResult("Failed to parse shader artifact header: " + path); + } + + const std::string magic(fileHeader.magic, fileHeader.magic + 7); + const bool isLegacySchema = magic == "XCSHD01" && fileHeader.schemaVersion == 1u; + const bool isSchemaV2 = magic == "XCSHD02" && fileHeader.schemaVersion == 2u; + const bool isSchemaV3 = magic == "XCSHD03" && fileHeader.schemaVersion == 3u; + const bool isCurrentSchema = + magic == "XCSHD04" && fileHeader.schemaVersion == kShaderArtifactSchemaVersion; + if (!isLegacySchema && !isSchemaV2 && !isSchemaV3 && !isCurrentSchema) { + return LoadResult("Invalid shader artifact header: " + path); + } + + auto shader = std::make_unique(); + + Containers::String shaderName; + Containers::String shaderSourcePath; + Containers::String shaderFallback; + if (!ReadShaderArtifactString(data, offset, shaderName) || + !ReadShaderArtifactString(data, offset, shaderSourcePath)) { + return LoadResult("Failed to parse shader artifact strings: " + path); + } + if (isCurrentSchema && + !ReadShaderArtifactString(data, offset, shaderFallback)) { + return LoadResult("Failed to parse shader artifact strings: " + path); + } + + shader->m_name = shaderName.Empty() ? path : shaderName; + shader->m_path = shaderSourcePath.Empty() ? path : shaderSourcePath; + shader->m_guid = ResourceGUID::Generate(shader->m_path); + shader->SetFallback(shaderFallback); + + ShaderArtifactHeader shaderHeader; + if (!ReadShaderArtifactValue(data, offset, shaderHeader)) { + return LoadResult("Failed to parse shader artifact body: " + path); + } + + for (Core::uint32 propertyIndex = 0; propertyIndex < shaderHeader.propertyCount; ++propertyIndex) { + ShaderPropertyDesc property = {}; + ShaderPropertyArtifact propertyArtifact; + if (!ReadShaderArtifactString(data, offset, property.name) || + !ReadShaderArtifactString(data, offset, property.displayName) || + !ReadShaderArtifactString(data, offset, property.defaultValue) || + !ReadShaderArtifactString(data, offset, property.semantic) || + !ReadShaderArtifactValue(data, offset, propertyArtifact)) { + return LoadResult("Failed to read shader artifact properties: " + path); + } + + property.type = static_cast(propertyArtifact.propertyType); + shader->AddProperty(property); + } + + for (Core::uint32 passIndex = 0; passIndex < shaderHeader.passCount; ++passIndex) { + Containers::String passName; + Core::uint32 tagCount = 0; + Core::uint32 resourceCount = 0; + Core::uint32 keywordDeclarationCount = 0; + Core::uint32 variantCount = 0; + Core::uint32 hasFixedFunctionState = 0; + MaterialRenderState fixedFunctionState = {}; + if (!ReadShaderArtifactString(data, offset, passName)) { + return LoadResult("Failed to read shader artifact passes: " + path); + } + + if (isLegacySchema) { + ShaderPassArtifactHeaderV1 passHeader = {}; + if (!ReadShaderArtifactValue(data, offset, passHeader)) { + return LoadResult("Failed to read shader artifact passes: " + path); + } + + tagCount = passHeader.tagCount; + resourceCount = passHeader.resourceCount; + variantCount = passHeader.variantCount; + } else if (isSchemaV2 || isSchemaV3) { + ShaderPassArtifactHeader passHeader = {}; + if (!ReadShaderArtifactValue(data, offset, passHeader)) { + return LoadResult("Failed to read shader artifact passes: " + path); + } + + tagCount = passHeader.tagCount; + resourceCount = passHeader.resourceCount; + keywordDeclarationCount = passHeader.keywordDeclarationCount; + variantCount = passHeader.variantCount; + } else { + ShaderPassArtifactHeaderV4 passHeader = {}; + if (!ReadShaderArtifactValue(data, offset, passHeader)) { + return LoadResult("Failed to read shader artifact passes: " + path); + } + + tagCount = passHeader.tagCount; + resourceCount = passHeader.resourceCount; + keywordDeclarationCount = passHeader.keywordDeclarationCount; + variantCount = passHeader.variantCount; + hasFixedFunctionState = passHeader.hasFixedFunctionState; + fixedFunctionState = passHeader.fixedFunctionState; + } + + ShaderPass pass = {}; + pass.name = passName; + pass.hasFixedFunctionState = hasFixedFunctionState != 0u; + pass.fixedFunctionState = fixedFunctionState; + shader->AddPass(pass); + + for (Core::uint32 tagIndex = 0; tagIndex < tagCount; ++tagIndex) { + Containers::String tagName; + Containers::String tagValue; + if (!ReadShaderArtifactString(data, offset, tagName) || + !ReadShaderArtifactString(data, offset, tagValue)) { + return LoadResult("Failed to read shader artifact pass tags: " + path); + } + + shader->SetPassTag(passName, tagName, tagValue); + } + + for (Core::uint32 resourceIndex = 0; resourceIndex < resourceCount; ++resourceIndex) { + ShaderResourceBindingDesc binding = {}; + ShaderResourceArtifact resourceArtifact; + if (!ReadShaderArtifactString(data, offset, binding.name) || + !ReadShaderArtifactString(data, offset, binding.semantic) || + !ReadShaderArtifactValue(data, offset, resourceArtifact)) { + return LoadResult("Failed to read shader artifact pass resources: " + path); + } + + binding.type = static_cast(resourceArtifact.resourceType); + binding.set = resourceArtifact.set; + binding.binding = resourceArtifact.binding; + shader->AddPassResourceBinding(passName, binding); + } + + for (Core::uint32 declarationIndex = 0; declarationIndex < keywordDeclarationCount; ++declarationIndex) { + ShaderKeywordDeclaration declaration = {}; + ShaderKeywordDeclarationArtifactHeader declarationHeader = {}; + if (!ReadShaderArtifactValue(data, offset, declarationHeader)) { + return LoadResult("Failed to read shader artifact pass keywords: " + path); + } + + declaration.type = + static_cast(declarationHeader.declarationType); + declaration.options.Reserve(declarationHeader.optionCount); + for (Core::uint32 optionIndex = 0; optionIndex < declarationHeader.optionCount; ++optionIndex) { + Containers::String option; + if (!ReadShaderArtifactString(data, offset, option)) { + return LoadResult("Failed to read shader artifact keyword options: " + path); + } + + declaration.options.PushBack(option); + } + + shader->AddPassKeywordDeclaration(passName, declaration); + } + + for (Core::uint32 variantIndex = 0; variantIndex < variantCount; ++variantIndex) { + ShaderStageVariant variant = {}; + Core::uint64 compiledBinarySize = 0; + Core::uint32 keywordCount = 0; + if (isCurrentSchema) { + ShaderVariantArtifactHeader variantHeader = {}; + if (!ReadShaderArtifactValue(data, offset, variantHeader)) { + return LoadResult("Failed to read shader artifact variants: " + path); + } + + variant.stage = static_cast(variantHeader.stage); + variant.language = static_cast(variantHeader.language); + variant.backend = static_cast(variantHeader.backend); + keywordCount = variantHeader.keywordCount; + compiledBinarySize = variantHeader.compiledBinarySize; + } else { + ShaderVariantArtifactHeaderV2 variantHeader = {}; + if (!ReadShaderArtifactValue(data, offset, variantHeader)) { + return LoadResult("Failed to read shader artifact variants: " + path); + } + + variant.stage = static_cast(variantHeader.stage); + variant.language = static_cast(variantHeader.language); + variant.backend = static_cast(variantHeader.backend); + compiledBinarySize = variantHeader.compiledBinarySize; + } + + if (!ReadShaderArtifactString(data, offset, variant.entryPoint) || + !ReadShaderArtifactString(data, offset, variant.profile) || + !ReadShaderArtifactString(data, offset, variant.sourceCode)) { + return LoadResult("Failed to read shader artifact variants: " + path); + } + + for (Core::uint32 keywordIndex = 0; keywordIndex < keywordCount; ++keywordIndex) { + Containers::String keyword; + if (!ReadShaderArtifactString(data, offset, keyword)) { + return LoadResult("Failed to read shader artifact variant keywords: " + path); + } + + variant.requiredKeywords.enabledKeywords.PushBack(keyword); + } + NormalizeShaderKeywordSetInPlace(variant.requiredKeywords); + + if (compiledBinarySize > 0) { + if (offset + compiledBinarySize > data.Size()) { + return LoadResult("Shader artifact variant binary payload is truncated: " + path); + } + + variant.compiledBinary.Resize(static_cast(compiledBinarySize)); + std::memcpy( + variant.compiledBinary.Data(), + data.Data() + offset, + static_cast(compiledBinarySize)); + offset += static_cast(compiledBinarySize); + } + + shader->AddPassVariant(passName, variant); + } + } + + if (shader->GetPassCount() == 1) { + const ShaderPass* defaultPass = shader->FindPass("Default"); + if (defaultPass != nullptr && defaultPass->variants.Size() == 1u) { + const ShaderStageVariant& variant = defaultPass->variants[0]; + if (variant.backend == ShaderBackend::Generic) { + shader->SetShaderType(variant.stage); + shader->SetShaderLanguage(variant.language); + shader->SetSourceCode(variant.sourceCode); + shader->SetCompiledBinary(variant.compiledBinary); + } + } + } + + shader->m_isValid = true; + shader->m_memorySize = CalculateShaderMemorySize(*shader); + return LoadResult(shader.release()); +} + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/src/Resources/Shader/ShaderArtifactLoader.h b/engine/src/Resources/Shader/ShaderArtifactLoader.h new file mode 100644 index 00000000..12f233e4 --- /dev/null +++ b/engine/src/Resources/Shader/ShaderArtifactLoader.h @@ -0,0 +1,12 @@ +#pragma once + +#include +#include + +namespace XCEngine { +namespace Resources { + +LoadResult LoadShaderArtifact(const Containers::String& path); + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/src/Resources/Shader/ShaderAuthoringLoader.cpp b/engine/src/Resources/Shader/ShaderAuthoringLoader.cpp new file mode 100644 index 00000000..545c74b2 --- /dev/null +++ b/engine/src/Resources/Shader/ShaderAuthoringLoader.cpp @@ -0,0 +1,102 @@ +#include "ShaderAuthoringLoader.h" + +#include "ShaderAuthoringParser.h" +#include "ShaderRuntimeBuildUtils.h" +#include "ShaderSourceUtils.h" + +#include + +namespace XCEngine { +namespace Resources { + +bool LooksLikeShaderAuthoring(const std::string& sourceText) { + return DetectShaderAuthoringStyle(sourceText) != ShaderAuthoringStyle::NotShaderAuthoring; +} + +bool CollectLegacyBackendSplitShaderDependencyPaths( + const Containers::String& path, + const std::string& sourceText, + Containers::Array& outDependencies) { + outDependencies.Clear(); + + ShaderIR shaderIR = {}; + Containers::String parseError; + if (!ParseLegacyBackendSplitShaderAuthoring(path, sourceText, shaderIR, &parseError)) { + return false; + } + + std::unordered_set seenPaths; + for (const ShaderSubShaderIR& subShader : shaderIR.subShaders) { + for (const ShaderPassIR& pass : subShader.passes) { + for (const ShaderBackendVariantIR& backendVariant : pass.backendVariants) { + const Containers::String resolvedVertexPath = + ResolveShaderDependencyPath(backendVariant.vertexSourcePath, path); + const std::string vertexKey = ToStdString(resolvedVertexPath); + if (!vertexKey.empty() && seenPaths.insert(vertexKey).second) { + outDependencies.PushBack(resolvedVertexPath); + } + + const Containers::String resolvedFragmentPath = + ResolveShaderDependencyPath(backendVariant.fragmentSourcePath, path); + const std::string fragmentKey = ToStdString(resolvedFragmentPath); + if (!fragmentKey.empty() && seenPaths.insert(fragmentKey).second) { + outDependencies.PushBack(resolvedFragmentPath); + } + } + } + } + + return true; +} + +bool CollectUnityStyleSingleSourceShaderDependencyPaths( + const Containers::String& path, + const std::string& sourceText, + Containers::Array& outDependencies) { + outDependencies.Clear(); + + ShaderIR shaderIR = {}; + Containers::String parseError; + if (!ParseUnityStyleSingleSourceShaderAuthoring(path, sourceText, shaderIR, &parseError)) { + return false; + } + + std::unordered_set seenPaths; + CollectQuotedIncludeDependencyPaths(path, shaderIR.sharedProgramSource, seenPaths, outDependencies); + for (const ShaderSubShaderIR& subShader : shaderIR.subShaders) { + CollectQuotedIncludeDependencyPaths(path, subShader.sharedProgramSource, seenPaths, outDependencies); + for (const ShaderPassIR& pass : subShader.passes) { + CollectQuotedIncludeDependencyPaths(path, pass.sharedProgramSource, seenPaths, outDependencies); + CollectQuotedIncludeDependencyPaths(path, pass.programSource, seenPaths, outDependencies); + } + } + + return true; +} + +LoadResult LoadLegacyBackendSplitShaderAuthoring( + const Containers::String& path, + const std::string& sourceText) { + ShaderIR shaderIR = {}; + Containers::String parseError; + if (!ParseLegacyBackendSplitShaderAuthoring(path, sourceText, shaderIR, &parseError)) { + return LoadResult(parseError); + } + + return BuildShaderFromIR(path, shaderIR); +} + +LoadResult LoadUnityStyleSingleSourceShaderAuthoring( + const Containers::String& path, + const std::string& sourceText) { + ShaderIR shaderIR = {}; + Containers::String parseError; + if (!ParseUnityStyleSingleSourceShaderAuthoring(path, sourceText, shaderIR, &parseError)) { + return LoadResult(parseError); + } + + return BuildShaderFromIR(path, shaderIR); +} + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/src/Resources/Shader/ShaderAuthoringLoader.h b/engine/src/Resources/Shader/ShaderAuthoringLoader.h new file mode 100644 index 00000000..ddb31561 --- /dev/null +++ b/engine/src/Resources/Shader/ShaderAuthoringLoader.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include + +#include + +namespace XCEngine { +namespace Resources { + +bool LooksLikeShaderAuthoring(const std::string& sourceText); + +bool CollectLegacyBackendSplitShaderDependencyPaths( + const Containers::String& path, + const std::string& sourceText, + Containers::Array& outDependencies); + +bool CollectUnityStyleSingleSourceShaderDependencyPaths( + const Containers::String& path, + const std::string& sourceText, + Containers::Array& outDependencies); + +LoadResult LoadLegacyBackendSplitShaderAuthoring( + const Containers::String& path, + const std::string& sourceText); + +LoadResult LoadUnityStyleSingleSourceShaderAuthoring( + const Containers::String& path, + const std::string& sourceText); + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/src/Resources/Shader/ShaderFileUtils.cpp b/engine/src/Resources/Shader/ShaderFileUtils.cpp new file mode 100644 index 00000000..164ad30a --- /dev/null +++ b/engine/src/Resources/Shader/ShaderFileUtils.cpp @@ -0,0 +1,98 @@ +#include "ShaderFileUtils.h" + +#include + +#include +#include + +namespace XCEngine { +namespace Resources { + +namespace { + +Containers::Array TryReadFileData( + const std::filesystem::path& filePath, + bool& opened) { + Containers::Array data; + + std::ifstream file(filePath, std::ios::binary | std::ios::ate); + if (!file.is_open()) { + opened = false; + return data; + } + + opened = true; + const std::streamsize size = file.tellg(); + if (size <= 0) { + return data; + } + + file.seekg(0, std::ios::beg); + data.Resize(static_cast(size)); + if (!file.read(reinterpret_cast(data.Data()), size)) { + data.Clear(); + } + + return data; +} + +} // namespace + +std::string ToStdStringFromBytes(const Containers::Array& data) { + return std::string(reinterpret_cast(data.Data()), data.Size()); +} + +Containers::Array ReadShaderFileData(const Containers::String& path) { + bool opened = false; + const std::filesystem::path inputPath(path.CStr()); + Containers::Array data = TryReadFileData(inputPath, opened); + if (opened || path.Empty() || inputPath.is_absolute()) { + return data; + } + + const Containers::String& resourceRoot = ResourceManager::Get().GetResourceRoot(); + if (resourceRoot.Empty()) { + return data; + } + + return TryReadFileData(std::filesystem::path(resourceRoot.CStr()) / inputPath, opened); +} + +bool ReadShaderTextFile(const Containers::String& path, Containers::String& outText) { + const Containers::Array data = ReadShaderFileData(path); + if (data.Empty()) { + return false; + } + + outText = ToStdStringFromBytes(data).c_str(); + return true; +} + +Containers::String GetShaderPathExtension(const Containers::String& path) { + size_t dotPos = Containers::String::npos; + for (size_t i = path.Length(); i > 0; --i) { + if (path[i - 1] == '.') { + dotPos = i - 1; + break; + } + } + + if (dotPos == Containers::String::npos) { + return Containers::String(); + } + + return path.Substring(dotPos + 1); +} + +ShaderType DetectShaderTypeFromPath(const Containers::String& path) { + const Containers::String ext = GetShaderPathExtension(path).ToLower(); + if (ext == "vert") return ShaderType::Vertex; + if (ext == "frag") return ShaderType::Fragment; + if (ext == "geom") return ShaderType::Geometry; + if (ext == "comp") return ShaderType::Compute; + + return ShaderType::Fragment; +} + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/src/Resources/Shader/ShaderFileUtils.h b/engine/src/Resources/Shader/ShaderFileUtils.h new file mode 100644 index 00000000..fc078bc3 --- /dev/null +++ b/engine/src/Resources/Shader/ShaderFileUtils.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace XCEngine { +namespace Resources { + +std::string ToStdStringFromBytes(const Containers::Array& data); + +Containers::Array ReadShaderFileData(const Containers::String& path); +bool ReadShaderTextFile(const Containers::String& path, Containers::String& outText); + +Containers::String GetShaderPathExtension(const Containers::String& path); +ShaderType DetectShaderTypeFromPath(const Containers::String& path); + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/src/Resources/Shader/ShaderLoader.cpp b/engine/src/Resources/Shader/ShaderLoader.cpp index cd72a0b6..4f1fda76 100644 --- a/engine/src/Resources/Shader/ShaderLoader.cpp +++ b/engine/src/Resources/Shader/ShaderLoader.cpp @@ -1,1237 +1,20 @@ #include -#include "ShaderAuthoringParser.h" -#include "ShaderSourceUtils.h" -#include -#include +#include "ShaderArtifactLoader.h" +#include "ShaderAuthoringLoader.h" +#include "ShaderAuthoringParser.h" +#include "ShaderFileUtils.h" +#include "ShaderManifestLoader.h" +#include "ShaderRuntimeBuildUtils.h" + #include #include -#include -#include -#include -#include -#include -#include -#include #include -#include -#include namespace XCEngine { namespace Resources { -namespace { - -std::string ToStdStringFromBytes(const Containers::Array& data) { - return std::string(reinterpret_cast(data.Data()), data.Size()); -} - -Containers::Array TryReadFileData( - const std::filesystem::path& filePath, - bool& opened) { - Containers::Array data; - - std::ifstream file(filePath, std::ios::binary | std::ios::ate); - if (!file.is_open()) { - opened = false; - return data; - } - - opened = true; - const std::streamsize size = file.tellg(); - if (size <= 0) { - return data; - } - - file.seekg(0, std::ios::beg); - data.Resize(static_cast(size)); - if (!file.read(reinterpret_cast(data.Data()), size)) { - data.Clear(); - } - - return data; -} - -Containers::Array ReadShaderFileData(const Containers::String& path) { - bool opened = false; - const std::filesystem::path inputPath(path.CStr()); - Containers::Array data = TryReadFileData(inputPath, opened); - if (opened || path.Empty() || inputPath.is_absolute()) { - return data; - } - - const Containers::String& resourceRoot = ResourceManager::Get().GetResourceRoot(); - if (resourceRoot.Empty()) { - return data; - } - - return TryReadFileData(std::filesystem::path(resourceRoot.CStr()) / inputPath, opened); -} - -Containers::String GetPathExtension(const Containers::String& path) { - size_t dotPos = Containers::String::npos; - for (size_t i = path.Length(); i > 0; --i) { - if (path[i - 1] == '.') { - dotPos = i - 1; - break; - } - } - - if (dotPos == Containers::String::npos) { - return Containers::String(); - } - - return path.Substring(dotPos + 1); -} - -bool FindValueStart(const std::string& json, const char* key, size_t& valuePos) { - const std::string token = std::string("\"") + key + "\""; - const size_t keyPos = json.find(token); - if (keyPos == std::string::npos) { - return false; - } - - const size_t colonPos = json.find(':', keyPos + token.length()); - if (colonPos == std::string::npos) { - return false; - } - - valuePos = SkipWhitespace(json, colonPos + 1); - return valuePos < json.size(); -} - -bool TryParseStringValue(const std::string& json, const char* key, Containers::String& outValue) { - size_t valuePos = 0; - if (!FindValueStart(json, key, valuePos)) { - return false; - } - - return ParseQuotedString(json, valuePos, outValue); -} - -bool TryExtractDelimitedValue( - const std::string& json, - const char* key, - char openChar, - char closeChar, - std::string& outValue) { - size_t valuePos = 0; - if (!FindValueStart(json, key, valuePos) || valuePos >= json.size() || json[valuePos] != openChar) { - return false; - } - - bool inString = false; - bool escaped = false; - int depth = 0; - - for (size_t pos = valuePos; pos < json.size(); ++pos) { - const char ch = json[pos]; - if (escaped) { - escaped = false; - continue; - } - - if (ch == '\\') { - escaped = true; - continue; - } - - if (ch == '"') { - inString = !inString; - continue; - } - - if (inString) { - continue; - } - - if (ch == openChar) { - ++depth; - } else if (ch == closeChar) { - --depth; - if (depth == 0) { - outValue = json.substr(valuePos, pos - valuePos + 1); - return true; - } - } - } - - return false; -} - -bool TryExtractObject(const std::string& json, const char* key, std::string& outObject) { - return TryExtractDelimitedValue(json, key, '{', '}', outObject); -} - -bool TryExtractArray(const std::string& json, const char* key, std::string& outArray) { - return TryExtractDelimitedValue(json, key, '[', ']', outArray); -} - -bool TryParseStringMapObject( - const std::string& objectText, - const std::function& onEntry) { - if (!onEntry || objectText.empty() || objectText.front() != '{' || objectText.back() != '}') { - return false; - } - - size_t pos = 1; - while (pos < objectText.size()) { - pos = SkipWhitespace(objectText, pos); - if (pos >= objectText.size()) { - return false; - } - - if (objectText[pos] == '}') { - return true; - } - - Containers::String key; - if (!ParseQuotedString(objectText, pos, key, &pos)) { - return false; - } - - pos = SkipWhitespace(objectText, pos); - if (pos >= objectText.size() || objectText[pos] != ':') { - return false; - } - - pos = SkipWhitespace(objectText, pos + 1); - Containers::String value; - if (!ParseQuotedString(objectText, pos, value, &pos)) { - return false; - } - - onEntry(key, value); - - pos = SkipWhitespace(objectText, pos); - if (pos >= objectText.size()) { - return false; - } - - if (objectText[pos] == ',') { - ++pos; - continue; - } - - if (objectText[pos] == '}') { - return true; - } - - return false; - } - - return false; -} - -bool SplitTopLevelArrayElements(const std::string& arrayText, std::vector& outElements) { - outElements.clear(); - if (arrayText.size() < 2 || arrayText.front() != '[' || arrayText.back() != ']') { - return false; - } - - bool inString = false; - bool escaped = false; - int objectDepth = 0; - int arrayDepth = 0; - size_t elementStart = std::string::npos; - - for (size_t pos = 1; pos + 1 < arrayText.size(); ++pos) { - const char ch = arrayText[pos]; - if (escaped) { - escaped = false; - continue; - } - - if (ch == '\\') { - escaped = true; - continue; - } - - if (ch == '"') { - if (elementStart == std::string::npos) { - elementStart = pos; - } - inString = !inString; - continue; - } - - if (inString) { - continue; - } - - if (std::isspace(static_cast(ch)) != 0) { - continue; - } - - if (elementStart == std::string::npos) { - elementStart = pos; - } - - if (ch == '{') { - ++objectDepth; - continue; - } - if (ch == '[') { - ++arrayDepth; - continue; - } - if (ch == '}') { - --objectDepth; - continue; - } - if (ch == ']') { - --arrayDepth; - continue; - } - - if (ch == ',' && objectDepth == 0 && arrayDepth == 0) { - if (elementStart != std::string::npos && pos > elementStart) { - outElements.push_back(TrimCopy(arrayText.substr(elementStart, pos - elementStart))); - } - elementStart = std::string::npos; - } - } - - if (elementStart != std::string::npos) { - const std::string tail = TrimCopy(arrayText.substr(elementStart, arrayText.size() - 1 - elementStart)); - if (!tail.empty()) { - outElements.push_back(tail); - } - } - - return true; -} - -bool TryParseShaderKeywordsArray( - const std::string& arrayText, - ShaderKeywordSet& outKeywordSet) { - std::vector keywordElements; - if (!SplitTopLevelArrayElements(arrayText, keywordElements)) { - return false; - } - - outKeywordSet = ShaderKeywordSet(); - outKeywordSet.enabledKeywords.Reserve(keywordElements.size()); - - for (const std::string& keywordElement : keywordElements) { - Containers::String keyword; - size_t nextPos = 0; - if (!ParseQuotedString(keywordElement, 0, keyword, &nextPos)) { - return false; - } - - if (SkipWhitespace(keywordElement, nextPos) != keywordElement.size()) { - return false; - } - - outKeywordSet.enabledKeywords.PushBack(keyword); - } - - NormalizeShaderKeywordSetInPlace(outKeywordSet); - return true; -} - -Containers::String GetDefaultEntryPoint(ShaderLanguage language, ShaderType stage) { - if (language != ShaderLanguage::HLSL) { - return Containers::String("main"); - } - - switch (stage) { - case ShaderType::Vertex: return "MainVS"; - case ShaderType::Fragment: return "MainPS"; - case ShaderType::Geometry: return "MainGS"; - case ShaderType::Compute: return "MainCS"; - case ShaderType::Hull: return "MainHS"; - case ShaderType::Domain: return "MainDS"; - default: return Containers::String(); - } -} - -Containers::String GetDefaultProfile( - ShaderLanguage language, - ShaderBackend backend, - ShaderType stage) { - if (language == ShaderLanguage::HLSL) { - switch (stage) { - case ShaderType::Vertex: return "vs_5_0"; - case ShaderType::Fragment: return "ps_5_0"; - case ShaderType::Geometry: return "gs_5_0"; - case ShaderType::Compute: return "cs_5_0"; - case ShaderType::Hull: return "hs_5_0"; - case ShaderType::Domain: return "ds_5_0"; - default: return Containers::String(); - } - } - - const bool isVulkan = backend == ShaderBackend::Vulkan; - switch (stage) { - case ShaderType::Vertex: - return isVulkan ? "vs_4_50" : "vs_4_30"; - case ShaderType::Fragment: - return isVulkan ? "fs_4_50" : "fs_4_30"; - case ShaderType::Geometry: - return isVulkan ? "gs_4_50" : "gs_4_30"; - case ShaderType::Compute: - return isVulkan ? "cs_4_50" : "cs_4_30"; - case ShaderType::Hull: - return isVulkan ? "hs_4_50" : "hs_4_30"; - case ShaderType::Domain: - return isVulkan ? "ds_4_50" : "ds_4_30"; - default: - return Containers::String(); - } -} - -bool ReadTextFile(const Containers::String& path, Containers::String& outText) { - const Containers::Array data = ReadShaderFileData(path); - if (data.Empty()) { - return false; - } - - outText = ToStdStringFromBytes(data).c_str(); - return true; -} - -size_t CalculateShaderMemorySize(const Shader& shader); - -LoadResult BuildShaderFromIR( - const Containers::String& path, - const ShaderIR& shaderIR) { - auto shader = std::make_unique(); - IResource::ConstructParams params; - params.path = path; - params.guid = ResourceGUID::Generate(path); - params.name = shaderIR.name; - shader->Initialize(params); - shader->SetFallback(shaderIR.fallback); - - for (const ShaderPropertyDesc& property : shaderIR.properties) { - shader->AddProperty(property); - } - - for (const ShaderSubShaderIR& subShader : shaderIR.subShaders) { - for (const ShaderPassIR& pass : subShader.passes) { - ShaderPass shaderPass = {}; - shaderPass.name = pass.name; - shaderPass.hasFixedFunctionState = pass.hasFixedFunctionState; - shaderPass.fixedFunctionState = pass.fixedFunctionState; - shader->AddPass(shaderPass); - - for (const ShaderTagIR& subShaderTag : subShader.tags) { - shader->SetPassTag(pass.name, subShaderTag.name, subShaderTag.value); - } - for (const ShaderTagIR& passTag : pass.tags) { - shader->SetPassTag(pass.name, passTag.name, passTag.value); - } - - for (const ShaderResourceBindingDesc& resourceBinding : pass.resources) { - shader->AddPassResourceBinding(pass.name, resourceBinding); - } - for (const ShaderKeywordDeclaration& keywordDeclaration : pass.keywordDeclarations) { - shader->AddPassKeywordDeclaration(pass.name, keywordDeclaration); - } - - if (!pass.backendVariants.empty()) { - const std::vector keywordSets = - BuildShaderKeywordVariantSets(pass.keywordDeclarations); - for (const ShaderBackendVariantIR& backendVariant : pass.backendVariants) { - Containers::String vertexSourceCode; - ShaderStageVariant vertexVariant = {}; - vertexVariant.stage = ShaderType::Vertex; - vertexVariant.backend = backendVariant.backend; - vertexVariant.language = backendVariant.language; - vertexVariant.entryPoint = - backendVariant.language == ShaderLanguage::HLSL && !pass.vertexEntryPoint.Empty() - ? pass.vertexEntryPoint - : GetDefaultEntryPoint(backendVariant.language, ShaderType::Vertex); - vertexVariant.profile = !backendVariant.vertexProfile.Empty() - ? backendVariant.vertexProfile - : GetDefaultProfile(backendVariant.language, backendVariant.backend, ShaderType::Vertex); - - const Containers::String resolvedVertexPath = - ResolveShaderDependencyPath(backendVariant.vertexSourcePath, path); - if (!ReadTextFile(resolvedVertexPath, vertexSourceCode)) { - return LoadResult("Failed to read shader authoring vertex source: " + resolvedVertexPath); - } - - Containers::String fragmentSourceCode; - ShaderStageVariant fragmentVariant = {}; - fragmentVariant.stage = ShaderType::Fragment; - fragmentVariant.backend = backendVariant.backend; - fragmentVariant.language = backendVariant.language; - fragmentVariant.entryPoint = - backendVariant.language == ShaderLanguage::HLSL && !pass.fragmentEntryPoint.Empty() - ? pass.fragmentEntryPoint - : GetDefaultEntryPoint(backendVariant.language, ShaderType::Fragment); - fragmentVariant.profile = !backendVariant.fragmentProfile.Empty() - ? backendVariant.fragmentProfile - : GetDefaultProfile(backendVariant.language, backendVariant.backend, ShaderType::Fragment); - - const Containers::String resolvedFragmentPath = - ResolveShaderDependencyPath(backendVariant.fragmentSourcePath, path); - if (!ReadTextFile(resolvedFragmentPath, fragmentSourceCode)) { - return LoadResult("Failed to read shader authoring fragment source: " + resolvedFragmentPath); - } - - for (const ShaderKeywordSet& keywordSet : keywordSets) { - vertexVariant.requiredKeywords = keywordSet; - vertexVariant.sourceCode = BuildKeywordVariantSource(vertexSourceCode, keywordSet); - shader->AddPassVariant(pass.name, vertexVariant); - - fragmentVariant.requiredKeywords = keywordSet; - fragmentVariant.sourceCode = BuildKeywordVariantSource(fragmentSourceCode, keywordSet); - shader->AddPassVariant(pass.name, fragmentVariant); - } - } - } else if (!pass.programSource.Empty()) { - Containers::String combinedSource; - AppendAuthoringSourceBlock(combinedSource, shaderIR.sharedProgramSource); - AppendAuthoringSourceBlock(combinedSource, subShader.sharedProgramSource); - AppendAuthoringSourceBlock(combinedSource, pass.sharedProgramSource); - AppendAuthoringSourceBlock(combinedSource, pass.programSource); - const Containers::String strippedCombinedSource = - StripUnityStyleAuthoringPragmas(combinedSource); - const std::vector keywordSets = - BuildShaderKeywordVariantSets(pass.keywordDeclarations); - - for (const ShaderKeywordSet& keywordSet : keywordSets) { - const Containers::String variantSource = - BuildKeywordVariantSource(strippedCombinedSource, keywordSet); - - ShaderStageVariant vertexVariant = {}; - vertexVariant.stage = ShaderType::Vertex; - vertexVariant.backend = ShaderBackend::Generic; - vertexVariant.language = ShaderLanguage::HLSL; - vertexVariant.requiredKeywords = keywordSet; - vertexVariant.entryPoint = - !pass.vertexEntryPoint.Empty() - ? pass.vertexEntryPoint - : GetDefaultEntryPoint(ShaderLanguage::HLSL, ShaderType::Vertex); - vertexVariant.profile = GetDefaultProfile( - ShaderLanguage::HLSL, - ShaderBackend::Generic, - ShaderType::Vertex); - vertexVariant.sourceCode = variantSource; - shader->AddPassVariant(pass.name, vertexVariant); - - ShaderStageVariant fragmentVariant = {}; - fragmentVariant.stage = ShaderType::Fragment; - fragmentVariant.backend = ShaderBackend::Generic; - fragmentVariant.language = ShaderLanguage::HLSL; - fragmentVariant.requiredKeywords = keywordSet; - fragmentVariant.entryPoint = - !pass.fragmentEntryPoint.Empty() - ? pass.fragmentEntryPoint - : GetDefaultEntryPoint(ShaderLanguage::HLSL, ShaderType::Fragment); - fragmentVariant.profile = GetDefaultProfile( - ShaderLanguage::HLSL, - ShaderBackend::Generic, - ShaderType::Fragment); - fragmentVariant.sourceCode = variantSource; - shader->AddPassVariant(pass.name, fragmentVariant); - } - } - } - } - - shader->m_memorySize = CalculateShaderMemorySize(*shader); - return LoadResult(shader.release()); -} - -bool LooksLikeShaderAuthoring(const std::string& sourceText) { - return DetectShaderAuthoringStyle(sourceText) != ShaderAuthoringStyle::NotShaderAuthoring; -} - -bool CollectLegacyBackendSplitShaderDependencyPaths( - const Containers::String& path, - const std::string& sourceText, - Containers::Array& outDependencies) { - outDependencies.Clear(); - - ShaderIR shaderIR = {}; - Containers::String parseError; - if (!ParseLegacyBackendSplitShaderAuthoring(path, sourceText, shaderIR, &parseError)) { - return false; - } - - std::unordered_set seenPaths; - for (const ShaderSubShaderIR& subShader : shaderIR.subShaders) { - for (const ShaderPassIR& pass : subShader.passes) { - for (const ShaderBackendVariantIR& backendVariant : pass.backendVariants) { - const Containers::String resolvedVertexPath = - ResolveShaderDependencyPath(backendVariant.vertexSourcePath, path); - const std::string vertexKey = ToStdString(resolvedVertexPath); - if (!vertexKey.empty() && seenPaths.insert(vertexKey).second) { - outDependencies.PushBack(resolvedVertexPath); - } - - const Containers::String resolvedFragmentPath = - ResolveShaderDependencyPath(backendVariant.fragmentSourcePath, path); - const std::string fragmentKey = ToStdString(resolvedFragmentPath); - if (!fragmentKey.empty() && seenPaths.insert(fragmentKey).second) { - outDependencies.PushBack(resolvedFragmentPath); - } - } - } - } - - return true; -} - -bool CollectUnityStyleSingleSourceShaderDependencyPaths( - const Containers::String& path, - const std::string& sourceText, - Containers::Array& outDependencies) { - outDependencies.Clear(); - - ShaderIR shaderIR = {}; - Containers::String parseError; - if (!ParseUnityStyleSingleSourceShaderAuthoring(path, sourceText, shaderIR, &parseError)) { - return false; - } - - std::unordered_set seenPaths; - CollectQuotedIncludeDependencyPaths(path, shaderIR.sharedProgramSource, seenPaths, outDependencies); - for (const ShaderSubShaderIR& subShader : shaderIR.subShaders) { - CollectQuotedIncludeDependencyPaths(path, subShader.sharedProgramSource, seenPaths, outDependencies); - for (const ShaderPassIR& pass : subShader.passes) { - CollectQuotedIncludeDependencyPaths(path, pass.sharedProgramSource, seenPaths, outDependencies); - CollectQuotedIncludeDependencyPaths(path, pass.programSource, seenPaths, outDependencies); - } - } - - return true; -} - -LoadResult LoadLegacyBackendSplitShaderAuthoring( - const Containers::String& path, - const std::string& sourceText) { - ShaderIR shaderIR = {}; - Containers::String parseError; - if (!ParseLegacyBackendSplitShaderAuthoring(path, sourceText, shaderIR, &parseError)) { - return LoadResult(parseError); - } - - return BuildShaderFromIR(path, shaderIR); -} - -LoadResult LoadUnityStyleSingleSourceShaderAuthoring( - const Containers::String& path, - const std::string& sourceText) { - ShaderIR shaderIR = {}; - Containers::String parseError; - if (!ParseUnityStyleSingleSourceShaderAuthoring(path, sourceText, shaderIR, &parseError)) { - return LoadResult(parseError); - } - - return BuildShaderFromIR(path, shaderIR); -} - -template -bool ReadShaderArtifactValue(const Containers::Array& data, size_t& offset, T& outValue) { - if (offset + sizeof(T) > data.Size()) { - return false; - } - - std::memcpy(&outValue, data.Data() + offset, sizeof(T)); - offset += sizeof(T); - return true; -} - -bool ReadShaderArtifactString(const Containers::Array& data, - size_t& offset, - Containers::String& outValue) { - Core::uint32 length = 0; - if (!ReadShaderArtifactValue(data, offset, length)) { - return false; - } - - if (length == 0) { - outValue.Clear(); - return true; - } - - if (offset + length > data.Size()) { - return false; - } - - outValue = Containers::String( - std::string(reinterpret_cast(data.Data() + offset), length).c_str()); - offset += length; - return true; -} - -bool TryParseUnsignedValue(const std::string& json, const char* key, Core::uint32& outValue) { - size_t valuePos = 0; - if (!FindValueStart(json, key, valuePos)) { - return false; - } - - size_t endPos = valuePos; - while (endPos < json.size() && std::isdigit(static_cast(json[endPos])) != 0) { - ++endPos; - } - - if (endPos == valuePos) { - return false; - } - - try { - outValue = static_cast(std::stoul(json.substr(valuePos, endPos - valuePos))); - return true; - } catch (...) { - return false; - } -} - -size_t CalculateShaderMemorySize(const Shader& shader) { - size_t memorySize = - sizeof(Shader) + shader.GetName().Length() + shader.GetPath().Length() + shader.GetFallback().Length(); - for (const ShaderPropertyDesc& property : shader.GetProperties()) { - memorySize += property.name.Length(); - memorySize += property.displayName.Length(); - memorySize += property.defaultValue.Length(); - memorySize += property.semantic.Length(); - } - for (const ShaderPass& pass : shader.GetPasses()) { - memorySize += pass.name.Length(); - for (const ShaderPassTagEntry& tag : pass.tags) { - memorySize += tag.name.Length(); - memorySize += tag.value.Length(); - } - for (const ShaderResourceBindingDesc& binding : pass.resources) { - memorySize += binding.name.Length(); - memorySize += binding.semantic.Length(); - } - for (const ShaderKeywordDeclaration& declaration : pass.keywordDeclarations) { - for (const Containers::String& option : declaration.options) { - memorySize += option.Length(); - } - } - for (const ShaderStageVariant& variant : pass.variants) { - for (const Containers::String& keyword : variant.requiredKeywords.enabledKeywords) { - memorySize += keyword.Length(); - } - memorySize += variant.entryPoint.Length(); - memorySize += variant.profile.Length(); - memorySize += variant.sourceCode.Length(); - memorySize += variant.compiledBinary.Size(); - } - } - - return memorySize; -} - -ShaderType DetectShaderTypeFromPath(const Containers::String& path) { - const Containers::String ext = GetPathExtension(path).ToLower(); - if (ext == "vert") return ShaderType::Vertex; - if (ext == "frag") return ShaderType::Fragment; - if (ext == "geom") return ShaderType::Geometry; - if (ext == "comp") return ShaderType::Compute; - - return ShaderType::Fragment; -} - -bool LooksLikeShaderManifest(const std::string& sourceText) { - const size_t firstContentPos = SkipWhitespace(sourceText, 0); - return firstContentPos < sourceText.size() && - sourceText[firstContentPos] == '{' && - sourceText.find("\"passes\"") != std::string::npos; -} - -bool CollectShaderManifestDependencyPaths(const Containers::String& path, - const std::string& jsonText, - Containers::Array& outDependencies) { - outDependencies.Clear(); - - std::string passesArray; - if (!TryExtractArray(jsonText, "passes", passesArray)) { - return false; - } - - std::vector passObjects; - if (!SplitTopLevelArrayElements(passesArray, passObjects)) { - return false; - } - - std::unordered_set seenPaths; - for (const std::string& passObject : passObjects) { - std::string variantsArray; - if (!TryExtractArray(passObject, "variants", variantsArray)) { - return false; - } - - std::vector variantObjects; - if (!SplitTopLevelArrayElements(variantsArray, variantObjects)) { - return false; - } - - for (const std::string& variantObject : variantObjects) { - Containers::String sourcePath; - if (!TryParseStringValue(variantObject, "source", sourcePath) && - !TryParseStringValue(variantObject, "sourcePath", sourcePath)) { - continue; - } - - const Containers::String resolvedPath = ResolveShaderDependencyPath(sourcePath, path); - const std::string key = ToStdString(resolvedPath); - if (!key.empty() && seenPaths.insert(key).second) { - outDependencies.PushBack(resolvedPath); - } - } - } - - return true; -} - -LoadResult LoadShaderManifest(const Containers::String& path, const std::string& jsonText) { - std::string passesArray; - if (!TryExtractArray(jsonText, "passes", passesArray)) { - return LoadResult("Shader manifest is missing a valid passes array: " + path); - } - - std::vector passObjects; - if (!SplitTopLevelArrayElements(passesArray, passObjects) || passObjects.empty()) { - return LoadResult("Shader manifest does not contain any pass objects: " + path); - } - - auto shader = std::make_unique(); - IResource::ConstructParams params; - params.path = path; - params.guid = ResourceGUID::Generate(path); - - Containers::String manifestName; - if (TryParseStringValue(jsonText, "name", manifestName) && !manifestName.Empty()) { - params.name = manifestName; - } else { - const std::filesystem::path shaderPath(path.CStr()); - const std::string stem = shaderPath.stem().generic_string(); - params.name = stem.empty() ? path : Containers::String(stem.c_str()); - } - - shader->Initialize(params); - - Containers::String fallback; - if (TryParseStringValue(jsonText, "fallback", fallback)) { - shader->SetFallback(fallback); - } - - std::string propertiesArray; - if (TryExtractArray(jsonText, "properties", propertiesArray)) { - std::vector propertyObjects; - if (!SplitTopLevelArrayElements(propertiesArray, propertyObjects)) { - return LoadResult("Shader manifest properties array could not be parsed: " + path); - } - - for (const std::string& propertyObject : propertyObjects) { - ShaderPropertyDesc property = {}; - if (!TryParseStringValue(propertyObject, "name", property.name) || property.name.Empty()) { - return LoadResult("Shader manifest property is missing a valid name: " + path); - } - - Containers::String propertyTypeName; - if (!TryParseStringValue(propertyObject, "type", propertyTypeName) || - !TryParseShaderPropertyType(propertyTypeName, property.type)) { - return LoadResult("Shader manifest property has an invalid type: " + path); - } - - if (!TryParseStringValue(propertyObject, "displayName", property.displayName)) { - property.displayName = property.name; - } - - if (!TryParseStringValue(propertyObject, "defaultValue", property.defaultValue)) { - TryParseStringValue(propertyObject, "default", property.defaultValue); - } - - TryParseStringValue(propertyObject, "semantic", property.semantic); - shader->AddProperty(property); - } - } - - for (const std::string& passObject : passObjects) { - Containers::String passName; - if (!TryParseStringValue(passObject, "name", passName) || passName.Empty()) { - return LoadResult("Shader manifest pass is missing a valid name: " + path); - } - - std::string tagsObject; - if (TryExtractObject(passObject, "tags", tagsObject)) { - if (!TryParseStringMapObject( - tagsObject, - [shaderPtr = shader.get(), &passName](const Containers::String& key, const Containers::String& value) { - shaderPtr->SetPassTag(passName, key, value); - })) { - return LoadResult("Shader manifest pass tags could not be parsed: " + path); - } - } - - std::string resourcesArray; - if (TryExtractArray(passObject, "resources", resourcesArray)) { - std::vector resourceObjects; - if (!SplitTopLevelArrayElements(resourcesArray, resourceObjects)) { - return LoadResult("Shader manifest pass resources could not be parsed: " + path); - } - - for (const std::string& resourceObject : resourceObjects) { - ShaderResourceBindingDesc resourceBinding = {}; - if (!TryParseStringValue(resourceObject, "name", resourceBinding.name) || - resourceBinding.name.Empty()) { - return LoadResult("Shader manifest pass resource is missing a valid name: " + path); - } - - Containers::String resourceTypeName; - if (!TryParseStringValue(resourceObject, "type", resourceTypeName) || - !TryParseShaderResourceType(resourceTypeName, resourceBinding.type)) { - return LoadResult("Shader manifest pass resource has an invalid type: " + path); - } - - if (!TryParseUnsignedValue(resourceObject, "set", resourceBinding.set)) { - return LoadResult("Shader manifest pass resource is missing a valid set: " + path); - } - if (!TryParseUnsignedValue(resourceObject, "binding", resourceBinding.binding)) { - return LoadResult("Shader manifest pass resource is missing a valid binding: " + path); - } - - TryParseStringValue(resourceObject, "semantic", resourceBinding.semantic); - shader->AddPassResourceBinding(passName, resourceBinding); - } - } - - std::string variantsArray; - if (!TryExtractArray(passObject, "variants", variantsArray)) { - return LoadResult("Shader manifest pass is missing variants: " + path); - } - - std::vector variantObjects; - if (!SplitTopLevelArrayElements(variantsArray, variantObjects) || variantObjects.empty()) { - return LoadResult("Shader manifest pass does not contain any variants: " + path); - } - - for (const std::string& variantObject : variantObjects) { - ShaderStageVariant variant = {}; - - Containers::String stageName; - if (!TryParseStringValue(variantObject, "stage", stageName) || - !TryParseShaderType(stageName, variant.stage)) { - return LoadResult("Shader manifest variant has an invalid stage: " + path); - } - - Containers::String backendName; - if (!TryParseStringValue(variantObject, "backend", backendName) || - !TryParseShaderBackend(backendName, variant.backend)) { - return LoadResult("Shader manifest variant has an invalid backend: " + path); - } - - Containers::String languageName; - if (!TryParseStringValue(variantObject, "language", languageName) || - !TryParseShaderLanguage(languageName, variant.language)) { - return LoadResult("Shader manifest variant has an invalid language: " + path); - } - - Containers::String sourceCode; - if (TryParseStringValue(variantObject, "sourceCode", sourceCode)) { - variant.sourceCode = sourceCode; - } else { - Containers::String sourcePath; - if (!TryParseStringValue(variantObject, "source", sourcePath) && - !TryParseStringValue(variantObject, "sourcePath", sourcePath)) { - return LoadResult("Shader manifest variant is missing source/sourceCode: " + path); - } - - const Containers::String resolvedSourcePath = ResolveShaderDependencyPath(sourcePath, path); - if (!ReadTextFile(resolvedSourcePath, variant.sourceCode)) { - return LoadResult("Failed to read shader variant source: " + resolvedSourcePath); - } - } - - if (!TryParseStringValue(variantObject, "entryPoint", variant.entryPoint)) { - variant.entryPoint = GetDefaultEntryPoint(variant.language, variant.stage); - } - - if (!TryParseStringValue(variantObject, "profile", variant.profile)) { - variant.profile = GetDefaultProfile(variant.language, variant.backend, variant.stage); - } - - std::string keywordsArray; - if (TryExtractArray(variantObject, "keywords", keywordsArray) && - !TryParseShaderKeywordsArray(keywordsArray, variant.requiredKeywords)) { - return LoadResult("Shader manifest variant keywords could not be parsed: " + path); - } - - shader->AddPassVariant(passName, variant); - } - } - - shader->m_memorySize = CalculateShaderMemorySize(*shader); - return LoadResult(shader.release()); -} - -LoadResult LoadShaderArtifact(const Containers::String& path) { - const Containers::Array data = ReadShaderFileData(path); - if (data.Empty()) { - return LoadResult("Failed to read shader artifact: " + path); - } - - size_t offset = 0; - ShaderArtifactFileHeader fileHeader; - if (!ReadShaderArtifactValue(data, offset, fileHeader)) { - return LoadResult("Failed to parse shader artifact header: " + path); - } - - const std::string magic(fileHeader.magic, fileHeader.magic + 7); - const bool isLegacySchema = magic == "XCSHD01" && fileHeader.schemaVersion == 1u; - const bool isSchemaV2 = magic == "XCSHD02" && fileHeader.schemaVersion == 2u; - const bool isSchemaV3 = magic == "XCSHD03" && fileHeader.schemaVersion == 3u; - const bool isCurrentSchema = - magic == "XCSHD04" && fileHeader.schemaVersion == kShaderArtifactSchemaVersion; - if (!isLegacySchema && !isSchemaV2 && !isSchemaV3 && !isCurrentSchema) { - return LoadResult("Invalid shader artifact header: " + path); - } - - auto shader = std::make_unique(); - - Containers::String shaderName; - Containers::String shaderSourcePath; - Containers::String shaderFallback; - if (!ReadShaderArtifactString(data, offset, shaderName) || - !ReadShaderArtifactString(data, offset, shaderSourcePath)) { - return LoadResult("Failed to parse shader artifact strings: " + path); - } - if (isCurrentSchema && - !ReadShaderArtifactString(data, offset, shaderFallback)) { - return LoadResult("Failed to parse shader artifact strings: " + path); - } - - shader->m_name = shaderName.Empty() ? path : shaderName; - shader->m_path = shaderSourcePath.Empty() ? path : shaderSourcePath; - shader->m_guid = ResourceGUID::Generate(shader->m_path); - shader->SetFallback(shaderFallback); - - ShaderArtifactHeader shaderHeader; - if (!ReadShaderArtifactValue(data, offset, shaderHeader)) { - return LoadResult("Failed to parse shader artifact body: " + path); - } - - for (Core::uint32 propertyIndex = 0; propertyIndex < shaderHeader.propertyCount; ++propertyIndex) { - ShaderPropertyDesc property = {}; - ShaderPropertyArtifact propertyArtifact; - if (!ReadShaderArtifactString(data, offset, property.name) || - !ReadShaderArtifactString(data, offset, property.displayName) || - !ReadShaderArtifactString(data, offset, property.defaultValue) || - !ReadShaderArtifactString(data, offset, property.semantic) || - !ReadShaderArtifactValue(data, offset, propertyArtifact)) { - return LoadResult("Failed to read shader artifact properties: " + path); - } - - property.type = static_cast(propertyArtifact.propertyType); - shader->AddProperty(property); - } - - for (Core::uint32 passIndex = 0; passIndex < shaderHeader.passCount; ++passIndex) { - Containers::String passName; - Core::uint32 tagCount = 0; - Core::uint32 resourceCount = 0; - Core::uint32 keywordDeclarationCount = 0; - Core::uint32 variantCount = 0; - Core::uint32 hasFixedFunctionState = 0; - MaterialRenderState fixedFunctionState = {}; - if (!ReadShaderArtifactString(data, offset, passName)) { - return LoadResult("Failed to read shader artifact passes: " + path); - } - - if (isLegacySchema) { - ShaderPassArtifactHeaderV1 passHeader = {}; - if (!ReadShaderArtifactValue(data, offset, passHeader)) { - return LoadResult("Failed to read shader artifact passes: " + path); - } - - tagCount = passHeader.tagCount; - resourceCount = passHeader.resourceCount; - variantCount = passHeader.variantCount; - } else if (isSchemaV2 || isSchemaV3) { - ShaderPassArtifactHeader passHeader = {}; - if (!ReadShaderArtifactValue(data, offset, passHeader)) { - return LoadResult("Failed to read shader artifact passes: " + path); - } - - tagCount = passHeader.tagCount; - resourceCount = passHeader.resourceCount; - keywordDeclarationCount = passHeader.keywordDeclarationCount; - variantCount = passHeader.variantCount; - } else { - ShaderPassArtifactHeaderV4 passHeader = {}; - if (!ReadShaderArtifactValue(data, offset, passHeader)) { - return LoadResult("Failed to read shader artifact passes: " + path); - } - - tagCount = passHeader.tagCount; - resourceCount = passHeader.resourceCount; - keywordDeclarationCount = passHeader.keywordDeclarationCount; - variantCount = passHeader.variantCount; - hasFixedFunctionState = passHeader.hasFixedFunctionState; - fixedFunctionState = passHeader.fixedFunctionState; - } - - ShaderPass pass = {}; - pass.name = passName; - pass.hasFixedFunctionState = hasFixedFunctionState != 0u; - pass.fixedFunctionState = fixedFunctionState; - shader->AddPass(pass); - - for (Core::uint32 tagIndex = 0; tagIndex < tagCount; ++tagIndex) { - Containers::String tagName; - Containers::String tagValue; - if (!ReadShaderArtifactString(data, offset, tagName) || - !ReadShaderArtifactString(data, offset, tagValue)) { - return LoadResult("Failed to read shader artifact pass tags: " + path); - } - - shader->SetPassTag(passName, tagName, tagValue); - } - - for (Core::uint32 resourceIndex = 0; resourceIndex < resourceCount; ++resourceIndex) { - ShaderResourceBindingDesc binding = {}; - ShaderResourceArtifact resourceArtifact; - if (!ReadShaderArtifactString(data, offset, binding.name) || - !ReadShaderArtifactString(data, offset, binding.semantic) || - !ReadShaderArtifactValue(data, offset, resourceArtifact)) { - return LoadResult("Failed to read shader artifact pass resources: " + path); - } - - binding.type = static_cast(resourceArtifact.resourceType); - binding.set = resourceArtifact.set; - binding.binding = resourceArtifact.binding; - shader->AddPassResourceBinding(passName, binding); - } - - for (Core::uint32 declarationIndex = 0; declarationIndex < keywordDeclarationCount; ++declarationIndex) { - ShaderKeywordDeclaration declaration = {}; - ShaderKeywordDeclarationArtifactHeader declarationHeader = {}; - if (!ReadShaderArtifactValue(data, offset, declarationHeader)) { - return LoadResult("Failed to read shader artifact pass keywords: " + path); - } - - declaration.type = - static_cast(declarationHeader.declarationType); - declaration.options.Reserve(declarationHeader.optionCount); - for (Core::uint32 optionIndex = 0; optionIndex < declarationHeader.optionCount; ++optionIndex) { - Containers::String option; - if (!ReadShaderArtifactString(data, offset, option)) { - return LoadResult("Failed to read shader artifact keyword options: " + path); - } - - declaration.options.PushBack(option); - } - - shader->AddPassKeywordDeclaration(passName, declaration); - } - - for (Core::uint32 variantIndex = 0; variantIndex < variantCount; ++variantIndex) { - ShaderStageVariant variant = {}; - Core::uint64 compiledBinarySize = 0; - Core::uint32 keywordCount = 0; - if (isCurrentSchema) { - ShaderVariantArtifactHeader variantHeader = {}; - if (!ReadShaderArtifactValue(data, offset, variantHeader)) { - return LoadResult("Failed to read shader artifact variants: " + path); - } - - variant.stage = static_cast(variantHeader.stage); - variant.language = static_cast(variantHeader.language); - variant.backend = static_cast(variantHeader.backend); - keywordCount = variantHeader.keywordCount; - compiledBinarySize = variantHeader.compiledBinarySize; - } else { - ShaderVariantArtifactHeaderV2 variantHeader = {}; - if (!ReadShaderArtifactValue(data, offset, variantHeader)) { - return LoadResult("Failed to read shader artifact variants: " + path); - } - - variant.stage = static_cast(variantHeader.stage); - variant.language = static_cast(variantHeader.language); - variant.backend = static_cast(variantHeader.backend); - compiledBinarySize = variantHeader.compiledBinarySize; - } - - if (!ReadShaderArtifactString(data, offset, variant.entryPoint) || - !ReadShaderArtifactString(data, offset, variant.profile) || - !ReadShaderArtifactString(data, offset, variant.sourceCode)) { - return LoadResult("Failed to read shader artifact variants: " + path); - } - - for (Core::uint32 keywordIndex = 0; keywordIndex < keywordCount; ++keywordIndex) { - Containers::String keyword; - if (!ReadShaderArtifactString(data, offset, keyword)) { - return LoadResult("Failed to read shader artifact variant keywords: " + path); - } - - variant.requiredKeywords.enabledKeywords.PushBack(keyword); - } - NormalizeShaderKeywordSetInPlace(variant.requiredKeywords); - - if (compiledBinarySize > 0) { - if (offset + compiledBinarySize > data.Size()) { - return LoadResult("Shader artifact variant binary payload is truncated: " + path); - } - - variant.compiledBinary.Resize(static_cast(compiledBinarySize)); - std::memcpy( - variant.compiledBinary.Data(), - data.Data() + offset, - static_cast(compiledBinarySize)); - offset += static_cast(compiledBinarySize); - } - - shader->AddPassVariant(passName, variant); - } - } - - if (shader->GetPassCount() == 1) { - const ShaderPass* defaultPass = shader->FindPass("Default"); - if (defaultPass != nullptr && defaultPass->variants.Size() == 1u) { - const ShaderStageVariant& variant = defaultPass->variants[0]; - if (variant.backend == ShaderBackend::Generic) { - shader->SetShaderType(variant.stage); - shader->SetShaderLanguage(variant.language); - shader->SetSourceCode(variant.sourceCode); - shader->SetCompiledBinary(variant.compiledBinary); - } - } - } - - shader->m_isValid = true; - shader->m_memorySize = CalculateShaderMemorySize(*shader); - return LoadResult(shader.release()); -} - -LoadResult LoadLegacySingleStageShader(const Containers::String& path, const std::string& sourceText) { - auto shader = std::make_unique(); - shader->m_path = path; - shader->m_name = path; - shader->m_guid = ResourceGUID::Generate(path); - - const Containers::String ext = GetPathExtension(path).ToLower(); - if (ext == "hlsl") { - shader->SetShaderLanguage(ShaderLanguage::HLSL); - } else { - shader->SetShaderLanguage(ShaderLanguage::GLSL); - } - - shader->SetShaderType(DetectShaderTypeFromPath(path)); - shader->SetSourceCode(sourceText.c_str()); - shader->m_isValid = true; - shader->m_memorySize = - sizeof(Shader) + - shader->m_name.Length() + - shader->m_path.Length() + - shader->GetSourceCode().Length(); - - return LoadResult(shader.release()); -} - -} // namespace - ShaderLoader::ShaderLoader() = default; ShaderLoader::~ShaderLoader() = default; @@ -1267,7 +50,7 @@ LoadResult ShaderLoader::Load(const Containers::String& path, const ImportSettin return CreateBuiltinShaderResource(path); } - const Containers::String ext = GetPathExtension(path).ToLower(); + const Containers::String ext = GetShaderPathExtension(path).ToLower(); if (ext == "xcshader") { return LoadShaderArtifact(path); } @@ -1300,15 +83,16 @@ ImportSettings* ShaderLoader::GetDefaultSettings() const { return nullptr; } -bool ShaderLoader::CollectSourceDependencies(const Containers::String& path, - Containers::Array& outDependencies) const { +bool ShaderLoader::CollectSourceDependencies( + const Containers::String& path, + Containers::Array& outDependencies) const { outDependencies.Clear(); if (IsBuiltinShaderPath(path)) { return true; } - const Containers::String ext = GetPathExtension(path).ToLower(); + const Containers::String ext = GetShaderPathExtension(path).ToLower(); if (ext != "shader") { return true; } diff --git a/engine/src/Resources/Shader/ShaderManifestLoader.cpp b/engine/src/Resources/Shader/ShaderManifestLoader.cpp new file mode 100644 index 00000000..15d6da06 --- /dev/null +++ b/engine/src/Resources/Shader/ShaderManifestLoader.cpp @@ -0,0 +1,518 @@ +#include "ShaderManifestLoader.h" + +#include "ShaderFileUtils.h" +#include "ShaderRuntimeBuildUtils.h" +#include "ShaderSourceUtils.h" + +#include +#include +#include +#include +#include +#include + +namespace XCEngine { +namespace Resources { + +namespace { + +bool FindValueStart(const std::string& json, const char* key, size_t& valuePos) { + const std::string token = std::string("\"") + key + "\""; + const size_t keyPos = json.find(token); + if (keyPos == std::string::npos) { + return false; + } + + const size_t colonPos = json.find(':', keyPos + token.length()); + if (colonPos == std::string::npos) { + return false; + } + + valuePos = SkipWhitespace(json, colonPos + 1); + return valuePos < json.size(); +} + +bool TryParseStringValue(const std::string& json, const char* key, Containers::String& outValue) { + size_t valuePos = 0; + if (!FindValueStart(json, key, valuePos)) { + return false; + } + + return ParseQuotedString(json, valuePos, outValue); +} + +bool TryExtractDelimitedValue( + const std::string& json, + const char* key, + char openChar, + char closeChar, + std::string& outValue) { + size_t valuePos = 0; + if (!FindValueStart(json, key, valuePos) || valuePos >= json.size() || json[valuePos] != openChar) { + return false; + } + + bool inString = false; + bool escaped = false; + int depth = 0; + + for (size_t pos = valuePos; pos < json.size(); ++pos) { + const char ch = json[pos]; + if (escaped) { + escaped = false; + continue; + } + + if (ch == '\\') { + escaped = true; + continue; + } + + if (ch == '"') { + inString = !inString; + continue; + } + + if (inString) { + continue; + } + + if (ch == openChar) { + ++depth; + } else if (ch == closeChar) { + --depth; + if (depth == 0) { + outValue = json.substr(valuePos, pos - valuePos + 1); + return true; + } + } + } + + return false; +} + +bool TryExtractObject(const std::string& json, const char* key, std::string& outObject) { + return TryExtractDelimitedValue(json, key, '{', '}', outObject); +} + +bool TryExtractArray(const std::string& json, const char* key, std::string& outArray) { + return TryExtractDelimitedValue(json, key, '[', ']', outArray); +} + +bool TryParseStringMapObject( + const std::string& objectText, + const std::function& onEntry) { + if (!onEntry || objectText.empty() || objectText.front() != '{' || objectText.back() != '}') { + return false; + } + + size_t pos = 1; + while (pos < objectText.size()) { + pos = SkipWhitespace(objectText, pos); + if (pos >= objectText.size()) { + return false; + } + + if (objectText[pos] == '}') { + return true; + } + + Containers::String key; + if (!ParseQuotedString(objectText, pos, key, &pos)) { + return false; + } + + pos = SkipWhitespace(objectText, pos); + if (pos >= objectText.size() || objectText[pos] != ':') { + return false; + } + + pos = SkipWhitespace(objectText, pos + 1); + Containers::String value; + if (!ParseQuotedString(objectText, pos, value, &pos)) { + return false; + } + + onEntry(key, value); + + pos = SkipWhitespace(objectText, pos); + if (pos >= objectText.size()) { + return false; + } + + if (objectText[pos] == ',') { + ++pos; + continue; + } + + if (objectText[pos] == '}') { + return true; + } + + return false; + } + + return false; +} + +bool SplitTopLevelArrayElements(const std::string& arrayText, std::vector& outElements) { + outElements.clear(); + if (arrayText.size() < 2 || arrayText.front() != '[' || arrayText.back() != ']') { + return false; + } + + bool inString = false; + bool escaped = false; + int objectDepth = 0; + int arrayDepth = 0; + size_t elementStart = std::string::npos; + + for (size_t pos = 1; pos + 1 < arrayText.size(); ++pos) { + const char ch = arrayText[pos]; + if (escaped) { + escaped = false; + continue; + } + + if (ch == '\\') { + escaped = true; + continue; + } + + if (ch == '"') { + if (elementStart == std::string::npos) { + elementStart = pos; + } + inString = !inString; + continue; + } + + if (inString) { + continue; + } + + if (std::isspace(static_cast(ch)) != 0) { + continue; + } + + if (elementStart == std::string::npos) { + elementStart = pos; + } + + if (ch == '{') { + ++objectDepth; + } else if (ch == '}') { + --objectDepth; + } else if (ch == '[') { + ++arrayDepth; + } else if (ch == ']') { + --arrayDepth; + } else if (ch == ',' && objectDepth == 0 && arrayDepth == 0) { + outElements.push_back(TrimCopy(arrayText.substr(elementStart, pos - elementStart))); + elementStart = std::string::npos; + } + } + + if (elementStart != std::string::npos) { + outElements.push_back(TrimCopy(arrayText.substr(elementStart, arrayText.size() - 1 - elementStart))); + } + + return true; +} + +bool TryParseShaderKeywordsArray( + const std::string& arrayText, + ShaderKeywordSet& outKeywordSet) { + outKeywordSet = {}; + + std::vector keywordElements; + if (!SplitTopLevelArrayElements(arrayText, keywordElements)) { + return false; + } + + for (const std::string& keywordElement : keywordElements) { + if (keywordElement.empty()) { + continue; + } + + Containers::String keyword; + size_t nextPos = 0; + if (!ParseQuotedString(keywordElement, 0, keyword, &nextPos)) { + return false; + } + + if (SkipWhitespace(keywordElement, nextPos) != keywordElement.size()) { + return false; + } + + outKeywordSet.enabledKeywords.PushBack(keyword); + } + + NormalizeShaderKeywordSetInPlace(outKeywordSet); + return true; +} + +bool TryParseUnsignedValue(const std::string& json, const char* key, Core::uint32& outValue) { + size_t valuePos = 0; + if (!FindValueStart(json, key, valuePos)) { + return false; + } + + size_t endPos = valuePos; + while (endPos < json.size() && std::isdigit(static_cast(json[endPos])) != 0) { + ++endPos; + } + + if (endPos == valuePos) { + return false; + } + + try { + outValue = static_cast(std::stoul(json.substr(valuePos, endPos - valuePos))); + return true; + } catch (...) { + return false; + } +} + +} // namespace + +bool LooksLikeShaderManifest(const std::string& sourceText) { + const size_t firstContentPos = SkipWhitespace(sourceText, 0); + return firstContentPos < sourceText.size() && + sourceText[firstContentPos] == '{' && + sourceText.find("\"passes\"") != std::string::npos; +} + +bool CollectShaderManifestDependencyPaths( + const Containers::String& path, + const std::string& jsonText, + Containers::Array& outDependencies) { + outDependencies.Clear(); + + std::string passesArray; + if (!TryExtractArray(jsonText, "passes", passesArray)) { + return false; + } + + std::vector passObjects; + if (!SplitTopLevelArrayElements(passesArray, passObjects)) { + return false; + } + + std::unordered_set seenPaths; + for (const std::string& passObject : passObjects) { + std::string variantsArray; + if (!TryExtractArray(passObject, "variants", variantsArray)) { + return false; + } + + std::vector variantObjects; + if (!SplitTopLevelArrayElements(variantsArray, variantObjects)) { + return false; + } + + for (const std::string& variantObject : variantObjects) { + Containers::String sourcePath; + if (!TryParseStringValue(variantObject, "source", sourcePath) && + !TryParseStringValue(variantObject, "sourcePath", sourcePath)) { + continue; + } + + const Containers::String resolvedPath = ResolveShaderDependencyPath(sourcePath, path); + const std::string key = ToStdString(resolvedPath); + if (!key.empty() && seenPaths.insert(key).second) { + outDependencies.PushBack(resolvedPath); + } + } + } + + return true; +} + +LoadResult LoadShaderManifest( + const Containers::String& path, + const std::string& jsonText) { + std::string passesArray; + if (!TryExtractArray(jsonText, "passes", passesArray)) { + return LoadResult("Shader manifest is missing a valid passes array: " + path); + } + + std::vector passObjects; + if (!SplitTopLevelArrayElements(passesArray, passObjects) || passObjects.empty()) { + return LoadResult("Shader manifest does not contain any pass objects: " + path); + } + + auto shader = std::make_unique(); + IResource::ConstructParams params; + params.path = path; + params.guid = ResourceGUID::Generate(path); + + Containers::String manifestName; + if (TryParseStringValue(jsonText, "name", manifestName) && !manifestName.Empty()) { + params.name = manifestName; + } else { + const std::filesystem::path shaderPath(path.CStr()); + const std::string stem = shaderPath.stem().generic_string(); + params.name = stem.empty() ? path : Containers::String(stem.c_str()); + } + + shader->Initialize(params); + + Containers::String fallback; + if (TryParseStringValue(jsonText, "fallback", fallback)) { + shader->SetFallback(fallback); + } + + std::string propertiesArray; + if (TryExtractArray(jsonText, "properties", propertiesArray)) { + std::vector propertyObjects; + if (!SplitTopLevelArrayElements(propertiesArray, propertyObjects)) { + return LoadResult("Shader manifest properties array could not be parsed: " + path); + } + + for (const std::string& propertyObject : propertyObjects) { + ShaderPropertyDesc property = {}; + if (!TryParseStringValue(propertyObject, "name", property.name) || property.name.Empty()) { + return LoadResult("Shader manifest property is missing a valid name: " + path); + } + + Containers::String propertyTypeName; + if (!TryParseStringValue(propertyObject, "type", propertyTypeName) || + !TryParseShaderPropertyType(propertyTypeName, property.type)) { + return LoadResult("Shader manifest property has an invalid type: " + path); + } + + if (!TryParseStringValue(propertyObject, "displayName", property.displayName)) { + property.displayName = property.name; + } + + if (!TryParseStringValue(propertyObject, "defaultValue", property.defaultValue)) { + TryParseStringValue(propertyObject, "default", property.defaultValue); + } + + TryParseStringValue(propertyObject, "semantic", property.semantic); + shader->AddProperty(property); + } + } + + for (const std::string& passObject : passObjects) { + Containers::String passName; + if (!TryParseStringValue(passObject, "name", passName) || passName.Empty()) { + return LoadResult("Shader manifest pass is missing a valid name: " + path); + } + + std::string tagsObject; + if (TryExtractObject(passObject, "tags", tagsObject)) { + if (!TryParseStringMapObject( + tagsObject, + [shaderPtr = shader.get(), &passName](const Containers::String& key, const Containers::String& value) { + shaderPtr->SetPassTag(passName, key, value); + })) { + return LoadResult("Shader manifest pass tags could not be parsed: " + path); + } + } + + std::string resourcesArray; + if (TryExtractArray(passObject, "resources", resourcesArray)) { + std::vector resourceObjects; + if (!SplitTopLevelArrayElements(resourcesArray, resourceObjects)) { + return LoadResult("Shader manifest pass resources could not be parsed: " + path); + } + + for (const std::string& resourceObject : resourceObjects) { + ShaderResourceBindingDesc resourceBinding = {}; + if (!TryParseStringValue(resourceObject, "name", resourceBinding.name) || + resourceBinding.name.Empty()) { + return LoadResult("Shader manifest pass resource is missing a valid name: " + path); + } + + Containers::String resourceTypeName; + if (!TryParseStringValue(resourceObject, "type", resourceTypeName) || + !TryParseShaderResourceType(resourceTypeName, resourceBinding.type)) { + return LoadResult("Shader manifest pass resource has an invalid type: " + path); + } + + if (!TryParseUnsignedValue(resourceObject, "set", resourceBinding.set)) { + return LoadResult("Shader manifest pass resource is missing a valid set: " + path); + } + if (!TryParseUnsignedValue(resourceObject, "binding", resourceBinding.binding)) { + return LoadResult("Shader manifest pass resource is missing a valid binding: " + path); + } + + TryParseStringValue(resourceObject, "semantic", resourceBinding.semantic); + shader->AddPassResourceBinding(passName, resourceBinding); + } + } + + std::string variantsArray; + if (!TryExtractArray(passObject, "variants", variantsArray)) { + return LoadResult("Shader manifest pass is missing variants: " + path); + } + + std::vector variantObjects; + if (!SplitTopLevelArrayElements(variantsArray, variantObjects) || variantObjects.empty()) { + return LoadResult("Shader manifest pass does not contain any variants: " + path); + } + + for (const std::string& variantObject : variantObjects) { + ShaderStageVariant variant = {}; + + Containers::String stageName; + if (!TryParseStringValue(variantObject, "stage", stageName) || + !TryParseShaderType(stageName, variant.stage)) { + return LoadResult("Shader manifest variant has an invalid stage: " + path); + } + + Containers::String backendName; + if (!TryParseStringValue(variantObject, "backend", backendName) || + !TryParseShaderBackend(backendName, variant.backend)) { + return LoadResult("Shader manifest variant has an invalid backend: " + path); + } + + Containers::String languageName; + if (!TryParseStringValue(variantObject, "language", languageName) || + !TryParseShaderLanguage(languageName, variant.language)) { + return LoadResult("Shader manifest variant has an invalid language: " + path); + } + + Containers::String sourceCode; + if (TryParseStringValue(variantObject, "sourceCode", sourceCode)) { + variant.sourceCode = sourceCode; + } else { + Containers::String sourcePath; + if (!TryParseStringValue(variantObject, "source", sourcePath) && + !TryParseStringValue(variantObject, "sourcePath", sourcePath)) { + return LoadResult("Shader manifest variant is missing source/sourceCode: " + path); + } + + const Containers::String resolvedSourcePath = ResolveShaderDependencyPath(sourcePath, path); + if (!ReadShaderTextFile(resolvedSourcePath, variant.sourceCode)) { + return LoadResult("Failed to read shader variant source: " + resolvedSourcePath); + } + } + + if (!TryParseStringValue(variantObject, "entryPoint", variant.entryPoint)) { + variant.entryPoint = GetDefaultEntryPoint(variant.language, variant.stage); + } + + if (!TryParseStringValue(variantObject, "profile", variant.profile)) { + variant.profile = GetDefaultProfile(variant.language, variant.backend, variant.stage); + } + + std::string keywordsArray; + if (TryExtractArray(variantObject, "keywords", keywordsArray) && + !TryParseShaderKeywordsArray(keywordsArray, variant.requiredKeywords)) { + return LoadResult("Shader manifest variant keywords could not be parsed: " + path); + } + + shader->AddPassVariant(passName, variant); + } + } + + shader->m_memorySize = CalculateShaderMemorySize(*shader); + return LoadResult(shader.release()); +} + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/src/Resources/Shader/ShaderManifestLoader.h b/engine/src/Resources/Shader/ShaderManifestLoader.h new file mode 100644 index 00000000..9212687d --- /dev/null +++ b/engine/src/Resources/Shader/ShaderManifestLoader.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include +#include + +#include + +namespace XCEngine { +namespace Resources { + +bool LooksLikeShaderManifest(const std::string& sourceText); + +bool CollectShaderManifestDependencyPaths( + const Containers::String& path, + const std::string& jsonText, + Containers::Array& outDependencies); + +LoadResult LoadShaderManifest( + const Containers::String& path, + const std::string& jsonText); + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/src/Resources/Shader/ShaderRuntimeBuildUtils.cpp b/engine/src/Resources/Shader/ShaderRuntimeBuildUtils.cpp new file mode 100644 index 00000000..61c015b4 --- /dev/null +++ b/engine/src/Resources/Shader/ShaderRuntimeBuildUtils.cpp @@ -0,0 +1,273 @@ +#include "ShaderRuntimeBuildUtils.h" + +#include "ShaderAuthoringParser.h" +#include "ShaderFileUtils.h" +#include "ShaderSourceUtils.h" + +#include + +namespace XCEngine { +namespace Resources { + +Containers::String GetDefaultEntryPoint(ShaderLanguage language, ShaderType stage) { + if (language == ShaderLanguage::HLSL) { + switch (stage) { + case ShaderType::Vertex: return "VSMain"; + case ShaderType::Fragment: return "PSMain"; + case ShaderType::Geometry: return "GSMain"; + case ShaderType::Compute: return "CSMain"; + case ShaderType::Hull: return "HSMain"; + case ShaderType::Domain: return "DSMain"; + default: return "main"; + } + } + + return "main"; +} + +Containers::String GetDefaultProfile( + ShaderLanguage language, + ShaderBackend backend, + ShaderType stage) { + if (language == ShaderLanguage::HLSL) { + switch (stage) { + case ShaderType::Vertex: return "vs_5_0"; + case ShaderType::Fragment: return "ps_5_0"; + case ShaderType::Geometry: return "gs_5_0"; + case ShaderType::Compute: return "cs_5_0"; + case ShaderType::Hull: return "hs_5_0"; + case ShaderType::Domain: return "ds_5_0"; + default: return Containers::String(); + } + } + + const bool isVulkan = backend == ShaderBackend::Vulkan; + switch (stage) { + case ShaderType::Vertex: + return isVulkan ? "vs_4_50" : "vs_4_30"; + case ShaderType::Fragment: + return isVulkan ? "fs_4_50" : "fs_4_30"; + case ShaderType::Geometry: + return isVulkan ? "gs_4_50" : "gs_4_30"; + case ShaderType::Compute: + return isVulkan ? "cs_4_50" : "cs_4_30"; + case ShaderType::Hull: + return isVulkan ? "hs_4_50" : "hs_4_30"; + case ShaderType::Domain: + return isVulkan ? "ds_4_50" : "ds_4_30"; + default: + return Containers::String(); + } +} + +size_t CalculateShaderMemorySize(const Shader& shader) { + size_t memorySize = + sizeof(Shader) + shader.GetName().Length() + shader.GetPath().Length() + shader.GetFallback().Length(); + for (const ShaderPropertyDesc& property : shader.GetProperties()) { + memorySize += property.name.Length(); + memorySize += property.displayName.Length(); + memorySize += property.defaultValue.Length(); + memorySize += property.semantic.Length(); + } + for (const ShaderPass& pass : shader.GetPasses()) { + memorySize += pass.name.Length(); + for (const ShaderPassTagEntry& tag : pass.tags) { + memorySize += tag.name.Length(); + memorySize += tag.value.Length(); + } + for (const ShaderResourceBindingDesc& binding : pass.resources) { + memorySize += binding.name.Length(); + memorySize += binding.semantic.Length(); + } + for (const ShaderKeywordDeclaration& declaration : pass.keywordDeclarations) { + for (const Containers::String& option : declaration.options) { + memorySize += option.Length(); + } + } + for (const ShaderStageVariant& variant : pass.variants) { + for (const Containers::String& keyword : variant.requiredKeywords.enabledKeywords) { + memorySize += keyword.Length(); + } + memorySize += variant.entryPoint.Length(); + memorySize += variant.profile.Length(); + memorySize += variant.sourceCode.Length(); + memorySize += variant.compiledBinary.Size(); + } + } + + return memorySize; +} + +LoadResult BuildShaderFromIR( + const Containers::String& path, + const ShaderIR& shaderIR) { + auto shader = std::make_unique(); + IResource::ConstructParams params; + params.path = path; + params.guid = ResourceGUID::Generate(path); + params.name = shaderIR.name; + shader->Initialize(params); + shader->SetFallback(shaderIR.fallback); + + for (const ShaderPropertyDesc& property : shaderIR.properties) { + shader->AddProperty(property); + } + + for (const ShaderSubShaderIR& subShader : shaderIR.subShaders) { + for (const ShaderPassIR& pass : subShader.passes) { + ShaderPass shaderPass = {}; + shaderPass.name = pass.name; + shaderPass.hasFixedFunctionState = pass.hasFixedFunctionState; + shaderPass.fixedFunctionState = pass.fixedFunctionState; + shader->AddPass(shaderPass); + + for (const ShaderTagIR& subShaderTag : subShader.tags) { + shader->SetPassTag(pass.name, subShaderTag.name, subShaderTag.value); + } + for (const ShaderTagIR& passTag : pass.tags) { + shader->SetPassTag(pass.name, passTag.name, passTag.value); + } + + for (const ShaderResourceBindingDesc& resourceBinding : pass.resources) { + shader->AddPassResourceBinding(pass.name, resourceBinding); + } + for (const ShaderKeywordDeclaration& keywordDeclaration : pass.keywordDeclarations) { + shader->AddPassKeywordDeclaration(pass.name, keywordDeclaration); + } + + if (!pass.backendVariants.empty()) { + const std::vector keywordSets = + BuildShaderKeywordVariantSets(pass.keywordDeclarations); + for (const ShaderBackendVariantIR& backendVariant : pass.backendVariants) { + Containers::String vertexSourceCode; + ShaderStageVariant vertexVariant = {}; + vertexVariant.stage = ShaderType::Vertex; + vertexVariant.backend = backendVariant.backend; + vertexVariant.language = backendVariant.language; + vertexVariant.entryPoint = + backendVariant.language == ShaderLanguage::HLSL && !pass.vertexEntryPoint.Empty() + ? pass.vertexEntryPoint + : GetDefaultEntryPoint(backendVariant.language, ShaderType::Vertex); + vertexVariant.profile = !backendVariant.vertexProfile.Empty() + ? backendVariant.vertexProfile + : GetDefaultProfile(backendVariant.language, backendVariant.backend, ShaderType::Vertex); + + const Containers::String resolvedVertexPath = + ResolveShaderDependencyPath(backendVariant.vertexSourcePath, path); + if (!ReadShaderTextFile(resolvedVertexPath, vertexSourceCode)) { + return LoadResult("Failed to read shader authoring vertex source: " + resolvedVertexPath); + } + + Containers::String fragmentSourceCode; + ShaderStageVariant fragmentVariant = {}; + fragmentVariant.stage = ShaderType::Fragment; + fragmentVariant.backend = backendVariant.backend; + fragmentVariant.language = backendVariant.language; + fragmentVariant.entryPoint = + backendVariant.language == ShaderLanguage::HLSL && !pass.fragmentEntryPoint.Empty() + ? pass.fragmentEntryPoint + : GetDefaultEntryPoint(backendVariant.language, ShaderType::Fragment); + fragmentVariant.profile = !backendVariant.fragmentProfile.Empty() + ? backendVariant.fragmentProfile + : GetDefaultProfile(backendVariant.language, backendVariant.backend, ShaderType::Fragment); + + const Containers::String resolvedFragmentPath = + ResolveShaderDependencyPath(backendVariant.fragmentSourcePath, path); + if (!ReadShaderTextFile(resolvedFragmentPath, fragmentSourceCode)) { + return LoadResult("Failed to read shader authoring fragment source: " + resolvedFragmentPath); + } + + for (const ShaderKeywordSet& keywordSet : keywordSets) { + vertexVariant.requiredKeywords = keywordSet; + vertexVariant.sourceCode = BuildKeywordVariantSource(vertexSourceCode, keywordSet); + shader->AddPassVariant(pass.name, vertexVariant); + + fragmentVariant.requiredKeywords = keywordSet; + fragmentVariant.sourceCode = BuildKeywordVariantSource(fragmentSourceCode, keywordSet); + shader->AddPassVariant(pass.name, fragmentVariant); + } + } + } else if (!pass.programSource.Empty()) { + Containers::String combinedSource; + AppendAuthoringSourceBlock(combinedSource, shaderIR.sharedProgramSource); + AppendAuthoringSourceBlock(combinedSource, subShader.sharedProgramSource); + AppendAuthoringSourceBlock(combinedSource, pass.sharedProgramSource); + AppendAuthoringSourceBlock(combinedSource, pass.programSource); + const Containers::String strippedCombinedSource = + StripUnityStyleAuthoringPragmas(combinedSource); + const std::vector keywordSets = + BuildShaderKeywordVariantSets(pass.keywordDeclarations); + + for (const ShaderKeywordSet& keywordSet : keywordSets) { + const Containers::String variantSource = + BuildKeywordVariantSource(strippedCombinedSource, keywordSet); + + ShaderStageVariant vertexVariant = {}; + vertexVariant.stage = ShaderType::Vertex; + vertexVariant.backend = ShaderBackend::Generic; + vertexVariant.language = ShaderLanguage::HLSL; + vertexVariant.requiredKeywords = keywordSet; + vertexVariant.entryPoint = + !pass.vertexEntryPoint.Empty() + ? pass.vertexEntryPoint + : GetDefaultEntryPoint(ShaderLanguage::HLSL, ShaderType::Vertex); + vertexVariant.profile = GetDefaultProfile( + ShaderLanguage::HLSL, + ShaderBackend::Generic, + ShaderType::Vertex); + vertexVariant.sourceCode = variantSource; + shader->AddPassVariant(pass.name, vertexVariant); + + ShaderStageVariant fragmentVariant = {}; + fragmentVariant.stage = ShaderType::Fragment; + fragmentVariant.backend = ShaderBackend::Generic; + fragmentVariant.language = ShaderLanguage::HLSL; + fragmentVariant.requiredKeywords = keywordSet; + fragmentVariant.entryPoint = + !pass.fragmentEntryPoint.Empty() + ? pass.fragmentEntryPoint + : GetDefaultEntryPoint(ShaderLanguage::HLSL, ShaderType::Fragment); + fragmentVariant.profile = GetDefaultProfile( + ShaderLanguage::HLSL, + ShaderBackend::Generic, + ShaderType::Fragment); + fragmentVariant.sourceCode = variantSource; + shader->AddPassVariant(pass.name, fragmentVariant); + } + } + } + } + + shader->m_memorySize = CalculateShaderMemorySize(*shader); + return LoadResult(shader.release()); +} + +LoadResult LoadLegacySingleStageShader( + const Containers::String& path, + const std::string& sourceText) { + auto shader = std::make_unique(); + shader->m_path = path; + shader->m_name = path; + shader->m_guid = ResourceGUID::Generate(path); + + const Containers::String ext = GetShaderPathExtension(path).ToLower(); + if (ext == "hlsl") { + shader->SetShaderLanguage(ShaderLanguage::HLSL); + } else { + shader->SetShaderLanguage(ShaderLanguage::GLSL); + } + + shader->SetShaderType(DetectShaderTypeFromPath(path)); + shader->SetSourceCode(sourceText.c_str()); + shader->m_isValid = true; + shader->m_memorySize = + sizeof(Shader) + + shader->m_name.Length() + + shader->m_path.Length() + + shader->GetSourceCode().Length(); + + return LoadResult(shader.release()); +} + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/src/Resources/Shader/ShaderRuntimeBuildUtils.h b/engine/src/Resources/Shader/ShaderRuntimeBuildUtils.h new file mode 100644 index 00000000..c1b8ba95 --- /dev/null +++ b/engine/src/Resources/Shader/ShaderRuntimeBuildUtils.h @@ -0,0 +1,30 @@ +#pragma once + +#include "ShaderIR.h" + +#include +#include + +#include + +namespace XCEngine { +namespace Resources { + +Containers::String GetDefaultEntryPoint(ShaderLanguage language, ShaderType stage); +Containers::String GetDefaultProfile( + ShaderLanguage language, + ShaderBackend backend, + ShaderType stage); + +size_t CalculateShaderMemorySize(const Shader& shader); + +LoadResult BuildShaderFromIR( + const Containers::String& path, + const ShaderIR& shaderIR); + +LoadResult LoadLegacySingleStageShader( + const Containers::String& path, + const std::string& sourceText); + +} // namespace Resources +} // namespace XCEngine