From 4c167bec0e4c16659f234b1d2bd2a8326c652bdf Mon Sep 17 00:00:00 2001 From: ssdfasd <2156608475@qq.com> Date: Thu, 2 Apr 2026 03:03:36 +0800 Subject: [PATCH] Implement initial Unity-style asset library cache --- ...nity式Library资产导入与缓存系统重构方案.md | 1595 +++++++++++++++++ editor/src/Managers/ProjectManager.cpp | 31 + engine/CMakeLists.txt | 6 + .../XCEngine/Components/MeshFilterComponent.h | 3 + .../Components/MeshRendererComponent.h | 3 + .../XCEngine/Core/Asset/ArtifactFormats.h | 59 + .../XCEngine/Core/Asset/AssetDatabase.h | 130 ++ .../include/XCEngine/Core/Asset/AssetGUID.h | 54 + engine/include/XCEngine/Core/Asset/AssetRef.h | 46 + .../XCEngine/Core/Asset/ResourceManager.h | 44 +- .../XCEngine/Core/Containers/HashMap.h | 14 + .../XCEngine/Resources/BuiltinResources.h | 34 + .../XCEngine/Resources/Material/Material.h | 35 +- engine/include/XCEngine/Resources/Resources.h | 1 + engine/src/Components/MeshFilterComponent.cpp | 54 +- .../src/Components/MeshRendererComponent.cpp | 98 +- engine/src/Core/Asset/AssetDatabase.cpp | 1089 +++++++++++ engine/src/Core/Asset/AssetGUID.cpp | 114 ++ engine/src/Core/Asset/ResourceManager.cpp | 88 +- engine/src/Core/IO/IResourceLoader.cpp | 43 +- engine/src/Resources/BuiltinResources.cpp | 677 +++++++ engine/src/Resources/Material/Material.cpp | 42 +- .../src/Resources/Material/MaterialLoader.cpp | 37 +- engine/src/Resources/Mesh/MeshLoader.cpp | 244 ++- .../src/Resources/Texture/TextureLoader.cpp | 54 +- .../test_mesh_render_components.cpp | 58 + .../Material/test_material_loader.cpp | 43 + tests/Resources/Mesh/test_mesh_loader.cpp | 102 ++ .../Resources/Texture/test_texture_loader.cpp | 93 + 29 files changed, 4818 insertions(+), 73 deletions(-) create mode 100644 docs/plan/Unity式Library资产导入与缓存系统重构方案.md create mode 100644 engine/include/XCEngine/Core/Asset/ArtifactFormats.h create mode 100644 engine/include/XCEngine/Core/Asset/AssetDatabase.h create mode 100644 engine/include/XCEngine/Core/Asset/AssetGUID.h create mode 100644 engine/include/XCEngine/Core/Asset/AssetRef.h create mode 100644 engine/include/XCEngine/Resources/BuiltinResources.h create mode 100644 engine/src/Core/Asset/AssetDatabase.cpp create mode 100644 engine/src/Core/Asset/AssetGUID.cpp create mode 100644 engine/src/Resources/BuiltinResources.cpp diff --git a/docs/plan/Unity式Library资产导入与缓存系统重构方案.md b/docs/plan/Unity式Library资产导入与缓存系统重构方案.md new file mode 100644 index 00000000..6473d7af --- /dev/null +++ b/docs/plan/Unity式Library资产导入与缓存系统重构方案.md @@ -0,0 +1,1595 @@ +# Unity式 Library 资产导入与缓存系统重构方案 + +## 0. 当前实施进度(2026-04-02) + +本节用于记录方案的实际落地状态,避免文档只停留在设计层。 + +### 0.1 阶段进度表 + +| 阶段 | 状态 | 当前结果 | +| --- | --- | --- | +| 阶段 0:基础类型与工程边界收口 | 已完成 | 已引入 `AssetGUID`、`AssetRef`,开始把运行时 `ResourceGUID` 与编辑器资产身份拆开。 | +| 阶段 1:`.meta` 与 SourceAssetDB | 已完成 | 已支持自动生成 `.meta`、扫描 `Assets`、建立 `guid/path/importer` 索引,并支持路径到 GUID/AssetRef 的反查。 | +| 阶段 2:Artifact Store 与 ArtifactDB | 已完成 | 已落地 `Library/SourceAssetDB`、`Library/ArtifactDB`、`Library/Artifacts`,支持 artifact key、导入失效判断与产物目录管理。 | +| 阶段 3:TextureImporter | 已完成 | 已支持纹理导入为 `xctex`,运行时可直接从 artifact 读取,不再总是解码原始图片。 | +| 阶段 4:ModelImporter | 已完成(初版) | 已支持模型导入为 `xcmesh`,并缓存模型关联纹理的 `xctex`;`ResourceManager` 会优先走 artifact。 | +| 阶段 5:Material 系统 lazy 引用化 | 部分完成 | 已补齐材质序列化所需的 tag/property/texture binding 访问接口,但仍是 eager 贴图加载,未完成真正的延迟纹理解析。 | +| 阶段 6:Scene / Component 引用迁移 | 部分完成 | `MeshFilterComponent` 与 `MeshRendererComponent` 已开始双写路径和 `AssetRef`,可兼容旧格式读取。Scene 全量格式迁移尚未完成。 | +| 阶段 7:异步导入与异步加载 | 未开始 | 当前导入与大资源加载仍发生在主线程,是仍会卡 editor 的直接原因。 | +| 阶段 8:清理、GC、工具与可视化 | 未开始 | 还没有 Reimport All、依赖图查看、orphan artifact 清理、导入面板等工具。 | + +### 0.2 本轮已经落地的具体内容 + +- 新增 `AssetGUID`、`AssetRef`、`ArtifactFormats`、`AssetDatabase`,正式建立 Unity 式资产身份与导入缓存基础设施。 +- `ResourceManager` 已接入 `AssetDatabase`,资源加载时会先尝试 `EnsureArtifact(...)`,命中后走 `Library` 产物。 +- `TextureLoader` 已支持 `xctex`,`MeshLoader` 已支持 `xcmesh`,从而把“导入”和“运行时加载”从代码路径上拆开。 +- `ProjectManager` 已支持忽略 `.meta` 文件显示,并在资源删除、移动、重命名时携带 `.meta` sidecar。 +- `MeshFilterComponent` 与 `MeshRendererComponent` 已开始从纯路径引用迁到 `AssetRef`,目前是“旧路径兼容 + 新引用双写”模式。 +- 已为纹理 artifact、模型 artifact、组件 `AssetRef` 序列化补充回归测试。 + +### 0.3 已验证结果 + +- `Debug` 下 `texture_tests.exe` 已通过,覆盖 `.meta` 自动生成、`Library` 产物生成、二次 `EnsureArtifact` 不重复导入、按 `AssetRef` 再加载纹理。 +- `Debug` 下 `components_tests.exe` 已通过,覆盖 `MeshRendererComponent` 的 `AssetRef` 双写序列化/反序列化,以及项目内材质按 `AssetRef` 恢复。 +- `Release` 下 `mesh_tests.exe` 已通过,覆盖 `obj + mtl + 贴图` 导入为 `xcmesh + xctex`、二次导入命中缓存、按 `AssetRef` 再加载模型。 +- `Release` 下 `XCEditor` 已构建通过。 + +### 0.4 当前版本仍然存在的限制 + +- 首次打开未缓存或失效的 `obj` 资源时,仍会在主线程同步执行导入,editor 会明显卡住。 +- 即使已经命中 artifact,当前模型 artifact 读取后仍会立即把关联贴图一起加载到内存,尚未做到材质纹理 lazy load。 +- 首帧渲染阶段仍会发生 GPU 资源创建与上传,这部分也会带来一次性卡顿。 +- `MaterialImporter` 还没有形成完整的独立 artifact 格式,材质系统仍处于过渡状态。 +- Scene 级别仍是“组件局部迁移”,还不是一套完整的新序列化协议。 + +## 1. 文档目的 + +本文档用于给 XCEngine 当前资源系统做一次正式的、可实施的重构设计,目标是把当前“打开场景时直接同步读取原始资源”的模式,重构为一套接近 Unity 的: + +- `Assets` 作为源资源目录 +- `.meta` 作为稳定资产身份与导入配置 +- `Library` 作为导入产物与索引缓存 +- 场景、组件、材质、渲染运行时全部通过稳定资产引用访问缓存产物 + +本文档不是一句“加个缓存”式的小修,而是资产身份、导入、缓存、运行时加载、组件引用、编辑器流程的系统级重构方案。 + +--- + +## 2. 这次重构要解决什么问题 + +### 2.1 当前最直接的问题 + +当前 editor 打开一个引用了 `obj` 模型的 scene 时,慢的不是 scene 文件本身,而是 scene 打开过程中同步触发了原始资源导入。 + +以当前 `project/Assets/Scenes/Backpack.xc` 为例,scene 文件本身很小,但里面的 `MeshFilter` 直接引用了: + +- `Assets/Models/backpack/backpack.obj` + +当前调用链是: + +1. SceneManager 打开 scene +2. Scene 反序列化 GameObject 与 Component +3. MeshFilterComponent 反序列化时,立刻按路径同步 `Load()` +4. MeshLoader 直接用 Assimp 读取原始 `obj` +5. MeshLoader 顺手把模型材质和贴图一起导入 +6. TextureLoader 直接读取并解码原始 `png/jpg` +7. 首帧渲染再把 CPU 资源上传到 GPU + +这意味着当前“打开 scene”其实包含了“首次导入资源”。 + +### 2.2 当前系统的结构性问题 + +当前系统慢,不是某一个函数写得慢,而是系统边界本身就不对。问题包括: + +- 资产身份不稳定。当前 `ResourceGUID` 本质上是路径 hash,不是独立资产 ID。文件改名、移动、目录调整都会导致身份变化。 +- Scene/Component 引用层错误。当前 scene 和组件直接存路径字符串,而不是稳定资产引用。 +- 资源导入和运行时加载混在一起。`IResourceLoader` 现在既承担“从源文件导入”,又承担“运行时拿到资源对象”。 +- 没有持久化导入缓存。当前没有等价于 Unity `Library/Artifacts` 的正式产物目录。 +- 没有源资源数据库。当前没有等价于 Unity `SourceAssetDB` 的资产身份与路径索引层。 +- 没有 artifact 数据库。当前没有等价于 Unity `ArtifactDB` 的导入结果索引层。 +- 没有依赖图。模型依赖哪些贴图、材质、侧车文件,目前没有正式记录。 +- 组件反序列化会立即加载资源。Scene 打开直接卡在主线程。 +- 异步加载器目前基本是空壳,不能承担真正的后台导入与后台加载。 +- 渲染时还有 editor 额外开销,例如 Scene View object-id pass 的首帧资源上传,但这属于次要问题。 + +### 2.3 为什么 Unity 看起来没这么卡 + +Unity 在打开 scene 时,通常不是重新解析 `obj/png/jpg` 源文件,而是: + +- 通过 `.meta` 的 `guid` 识别资产 +- 通过 `Library` 中已经存在的导入产物恢复资源 +- 只有源文件、导入设置、依赖变化时,才重新导入 + +也就是说: + +- Unity 把“导入”提前做了 +- 你当前的 editor 把“导入”塞进了“打开场景” + +这就是 Unity 里大场景不一定很卡,而你这里小 scene 也会卡的根本原因。 + +--- + +## 3. 本地 Unity 项目观察结果 + +本次方案参考了本地 Unity 项目: + +- `D:\Xuanchi\Main\Editor` + +从该项目可以明确观察到以下结构: + +### 3.1 `.meta` 文件 + +Unity 的每个源资源旁边都有 `.meta` 文件,最核心字段是: + +- `guid` +- importer 类型 +- importer 设置 + +例如 `backpack.obj.meta` 中可以看到: + +- `guid` +- `ModelImporter` +- 材质导入模式 +- mesh 导入配置 +- tangent/normal 配置 + +这说明 Unity 的源资源身份和导入配置是放在源资源旁的,而不是临时内存状态。 + +### 3.2 `Library` 的职责分层 + +在本地 Unity 项目里,`Library` 下可以看到: + +- `SourceAssetDB` +- `ArtifactDB` +- `Artifacts` +- `ShaderCache` +- `ScriptAssemblies` +- `StateCache` + +这说明 Unity 的 `Library` 不是一个单独的缓存目录,而是多个子系统缓存的集合。 + +### 3.3 本次只模仿哪一部分 + +本次 XCEngine 初版只模仿 Unity 的“资产导入与缓存”这一部分,不复制以下内容: + +- 脚本编译缓存 +- 包管理缓存 +- build 缓存 +- editor 状态缓存 +- shader 编译缓存 + +本次重点只放在: + +- `Assets` +- `.meta` +- `SourceAssetDB` +- `ArtifactDB` +- `Artifacts` + +--- + +## 4. 重构目标 + +### 4.1 总目标 + +把当前资源系统重构为: + +`源文件系统`、`资产身份系统`、`导入系统`、`artifact 缓存系统`、`运行时资源加载系统`、`场景资产引用系统` + +六层明确分离的正式资产管线。 + +### 4.2 直接目标 + +- 打开 scene 时不再直接读取原始 `obj/png/jpg` +- 首次导入后,后续打开 scene 优先命中 `Library` artifact +- 文件改名/移动后,scene 引用不失效 +- 模型、材质、贴图引用不再依赖路径字符串 +- 模型导入不再把所有外部贴图同步塞进 `Mesh` 对象里 +- 资源导入的失效条件可控、可追踪、可重建 + +### 4.3 长期目标 + +- 为后续 prefab、动画、shader、音频、打包系统留出统一入口 +- 为后续后台导入、导入进度 UI、资产依赖分析、项目迁移工具提供基础设施 +- 为后续 player 构建系统提供统一的“已导入资源来源” + +--- + +## 5. 非目标 + +这次重构的第一版不追求: + +- 100% 复制 Unity 内部数据库格式 +- 立即实现所有资源类型的 importer +- 一步到位做完脚本编译缓存与 ShaderCache +- 一次性改完整个 editor 的所有资产面板功能 +- 一开始就做压缩纹理格式转码、平台分包、远程缓存 + +第一版必须先把“资产导入缓存”这条主链路做对。 + +--- + +## 6. 设计原则 + +### 6.1 资产身份必须独立于路径 + +路径只是定位信息,不应该承担身份。 + +新的系统必须引入独立 `AssetGUID`,并把它落盘到 `.meta`。改名和移动只影响路径索引,不影响资产身份。 + +### 6.2 导入和运行时加载必须分离 + +导入器负责: + +- 读取原始源文件 +- 生成规范化中间产物 +- 记录依赖 +- 生成 artifact + +运行时加载器负责: + +- 读取 artifact +- 创建运行时资源对象 +- 加入内存 cache + +不能再让一个 `Load(path)` 同时承担两件事。 + +### 6.3 Scene 反序列化不能立即触发重型资源导入 + +Scene 打开时应该只恢复引用关系和组件数据,不应在反序列化过程中同步读取大模型和大贴图源文件。 + +### 6.4 运行时资源对象不能再“顺手拥有所有关联资源” + +当前最大的结构性浪费之一是: + +- `Mesh` 导入时直接把材质和贴图都挂进去 +- `Material` 运行时对象也直接持有纹理 handle + +这样会导致: + +- 只需要 mesh 时,材质和贴图也跟着进来 +- 只渲染 base color 时,normal/specular/roughness/ao 也同步加载 + +重构后应改成: + +- `Mesh` 只管 mesh 数据和默认材质引用 +- `Material` 只管材质属性和纹理引用 +- 纹理运行时加载按需触发 + +### 6.5 缓存键必须可重复计算 + +artifact 是否有效,必须由可重复计算的输入决定,而不是靠“我觉得没变”。 + +artifact key 必须包含: + +- 源文件内容 hash +- 导入设置 hash +- importer version +- 依赖文件/依赖资产 hash +- 平台或目标格式 + +--- + +## 7. 新系统的核心概念 + +### 7.1 AssetGUID + +新增稳定资产 ID。 + +定义建议: + +- 128-bit +- 文本形式为 32 位小写十六进制 +- 在 `.meta` 首次生成时随机创建 +- 之后不再因为路径变化而变化 + +注意: + +- 当前 `ResourceGUID` 不是这个概念 +- 当前 `ResourceGUID` 可以暂时保留给运行时 cache 用 +- `AssetGUID` 用于编辑器和资产引用层 + +### 7.2 LocalID + +一个源资产导入后,可能产生多个子资源。 + +例如一个 `obj` 模型可能产生: + +- 主模型资产 +- 一个或多个 `Mesh` +- 一个或多个默认 `Material` +- 内嵌纹理子资产 + +因此需要 `LocalID` 表示“某个资产内部的某个子对象”。 + +定义建议: + +- 64-bit +- 在 importer 内根据“子资源逻辑路径”稳定生成 +- 不依赖内存地址 +- 尽量在重导入后保持稳定 + +### 7.3 AssetRef + +新的统一资产引用结构: + +```text +AssetRef = { assetGuid, localID, resourceType } +``` + +所有 scene、prefab、material、component 最终都应该依赖这个结构,而不是路径字符串。 + +### 7.4 SourceAssetDB + +源资产数据库,用来记录: + +- `AssetGUID -> 源路径` +- `源路径 -> AssetGUID` +- `.meta` 内容 +- importer 类型 +- importer 设置 hash +- 源文件指纹 +- 是否为文件/文件夹 +- 上次导入状态 + +### 7.5 ArtifactKey + +导入产物的内容键。 + +它不是资产 GUID,而是本次“导入结果”的键。 + +同一个 `AssetGUID` 在不同 importer 设置或不同依赖版本下,会产生不同 `ArtifactKey`。 + +### 7.6 ArtifactDB + +导入结果数据库,用来记录: + +- `(AssetGUID, importer signature) -> ArtifactKey` +- `ArtifactKey -> 产物清单` +- `ArtifactKey -> 依赖集合` +- 主子资源映射 +- sub-asset `LocalID -> artifact object` + +### 7.7 Artifact Store + +真正落盘的导入产物目录,对应 Unity 的 `Library/Artifacts`。 + +### 7.8 Importer + +新的导入器不再等于运行时 loader,而是只负责: + +- 从源资源生成 artifact +- 声明依赖 +- 产出 sub-asset 映射 + +### 7.9 Runtime Artifact Loader + +新的运行时资源加载器负责: + +- 从 artifact 读取 `xcmesh/xctex/xcmat/...` +- 创建 `Mesh/Texture/Material` 等运行时对象 +- 写入 `ResourceManager` + +--- + +## 8. 总体架构 + +### 8.1 目标数据流 + +```text +Assets + .meta + -> +Project Asset Scan + -> +SourceAssetDB + -> +Importer Registry + -> +ArtifactDB + Artifact Store + -> +Runtime Artifact Loader + -> +ResourceManager + -> +Scene / Component / Renderer +``` + +### 8.2 目标职责边界 + +#### 源文件层 + +只负责: + +- `Assets` 下的真实源文件 +- 同路径 `.meta` + +#### 编辑器资产数据库层 + +只负责: + +- 维护 `AssetGUID` +- 管理路径与 meta +- 决定是否需要 reimport +- 维护依赖索引 + +#### 导入层 + +只负责: + +- 从源文件读取 +- 生成 artifact +- 写入 `Library` + +#### 运行时资源层 + +只负责: + +- 读取 artifact +- 构造运行时资源对象 +- 做内存 cache + +#### Scene/Component 层 + +只负责: + +- 保存 `AssetRef` +- 请求资源 +- 不直接操作源文件 + +--- + +## 9. 推荐的目录结构 + +建议项目目录最终如下: + +```text +project/ +├── Assets/ +│ ├── Scenes/ +│ ├── Models/ +│ ├── Textures/ +│ ├── Materials/ +│ └── ... + 对应 .meta +├── Library/ +│ ├── SourceAssetDB/ +│ │ ├── assets.json +│ │ ├── guid_index.json +│ │ ├── path_index.json +│ │ └── folders.json +│ ├── ArtifactDB/ +│ │ ├── artifacts.json +│ │ ├── asset_to_artifact.json +│ │ ├── dependency_index.json +│ │ └── subasset_index.json +│ ├── Artifacts/ +│ │ ├── 00/ +│ │ ├── 01/ +│ │ ├── ... +│ │ └── ff/ +│ ├── ImportLogs/ +│ └── CacheVersion.json +├── .xceditor/ +│ └── imgui_layout.ini +└── Project.xcproject +``` + +### 9.1 为什么初版不直接做成 Unity 那种单文件 DB + +Unity 的 `SourceAssetDB` 与 `ArtifactDB` 在本地看起来更像单文件数据库。 + +但 XCEngine 初版建议先做成: + +- 同名目录 +- 多个版本化 JSON 索引文件 + +理由: + +- 易实现 +- 易调试 +- 易人工修复 +- 易写测试 +- 后续可以在不改外部结构的前提下替换为二进制 DB 或 SQLite + +也就是说: + +- 逻辑结构模仿 Unity +- 存储后端先选工程上可落地的版本 + +--- + +## 10. `.meta` 系统设计 + +### 10.1 总体目标 + +每个 `Assets` 内的文件和文件夹都应该有 `.meta`。 + +`.meta` 要承担: + +- 稳定 `AssetGUID` +- importer 类型 +- importer 设置 +- importer 版本 +- 用户数据与扩展字段 + +### 10.2 初版格式建议 + +为了最大程度贴近 Unity,推荐采用“Unity 风格文本 + 严格子集解析器”的方式,而不是纯 JSON。 + +示例: + +```yaml +fileFormatVersion: 1 +guid: 07215647f41c2504abc024d70182ba63 +folderAsset: false +importer: ModelImporter +importerVersion: 1 +userData: +labels: [] +ModelImporter: + materials: + importMode: ImportAsSubAssets + searchMode: LocalDirectory + meshes: + globalScale: 1.0 + generateNormals: FromSource + generateTangents: IfNeeded + importCameras: false + importLights: false + weldVertices: true +``` + +### 10.3 为什么不直接上完整 YAML + +完整 YAML 解析超出当前工程必要范围。 + +初版建议: + +- 文本表现模仿 Unity +- 解析器只支持我们定义的固定字段子集 +- 统一由 editor 写回,尽量不手写任意结构 + +### 10.4 `.meta` 的生成规则 + +- `Assets` 下扫描到新文件/文件夹时,若缺少 `.meta`,自动生成 +- 新生成 `.meta` 时写入随机 `AssetGUID` +- 重复 GUID 检测到时,自动再生并记录错误日志 +- 文件重命名、移动时 `.meta` 跟随文件一起移动 +- 文件夹同样有 `.meta` + +--- + +## 11. 资产数据库设计 + +### 11.1 SourceAssetDB 记录结构 + +建议的 `SourceAssetRecord`: + +```text +guid +relativePath +metaPath +isFolder +importerName +importerVersion +metaHash +sourceHash +sourceFileSize +sourceWriteTime +importSettingsHash +lastKnownArtifactKey +state +``` + +其中: + +- `sourceHash` 用于真正判断内容是否变化 +- `sourceFileSize + sourceWriteTime` 用作快速短路 +- `metaHash` 用于判断 importer 设置变化 + +### 11.2 ArtifactDB 记录结构 + +建议的 `ArtifactRecord`: + +```text +artifactKey +assetGuid +importerName +importerVersion +targetProfile +inputSignatureHash +artifactPath +mainObjectLocalID +subAssets[] +dependencies[] +createdAt +``` + +### 11.3 依赖记录结构 + +依赖应支持两种: + +- 源文件依赖 +- 资产依赖 + +建议字段: + +```text +dependencyKind = SourceFile | Asset +sourcePath / assetGuid +expectedHash +``` + +--- + +## 12. 哈希与键设计 + +### 12.1 AssetGUID + +用途: + +- 标识“这个资产是谁” + +生成方式: + +- 首次生成时随机 UUID 风格 128-bit + +### 12.2 ContentHash + +用途: + +- 标识“这个文件内容是什么” + +建议实现: + +- 引入 `xxHash` 的 `XXH3_128` +- 用于源文件内容 hash、meta 文本 hash、artifact 输入签名 hash + +### 12.3 ArtifactKey + +用途: + +- 标识“这次导入结果是什么” + +建议组成: + +```text +ArtifactKey = Hash128( + importerName, + importerVersion, + assetGuid, + sourceContentHash, + importSettingsHash, + dependencySignatureHash, + targetProfile, + artifactSchemaVersion +) +``` + +### 12.4 Runtime Resource GUID + +当前 `ResourceGUID` 不应再直接由源路径生成。 + +重构后建议: + +- `ResourceGUID` 用于运行时对象 cache +- 由 `artifactKey + localID + resourceType` 生成 + +这样: + +- 资产改名不会改变运行时对象身份 +- 同一 artifact 的 sub-asset 可以稳定寻址 + +--- + +## 13. Artifact Store 设计 + +### 13.1 落盘组织方式 + +建议采用哈希分片目录: + +```text +Library/Artifacts/07/07ab23cd.../ +``` + +目录名是完整 `artifactKey`,前两位做分片。 + +### 13.2 每个 artifact 包内的内容 + +建议不要做成一个不可读 blob,而是一个小型包目录: + +```text +Library/Artifacts/07/07ab23cd.../ +├── artifact.json +├── main.xcasset +├── mesh_1001.xcmesh +├── material_2001.xcmat +├── material_2002.xcmat +└── embedded_texture_3001.xctex +``` + +这样做的原因: + +- 易调试 +- 易增量扩展 +- 易人工清理 +- 适合初版实现 + +### 13.3 artifact.json 内容 + +建议至少包含: + +- `artifactKey` +- `assetGuid` +- `importer` +- `importerVersion` +- `schemaVersion` +- `mainObject` +- `subAssets` +- `dependencies` +- `producedFiles` + +--- + +## 14. Importer 体系重构 + +### 14.1 现有问题 + +当前 `MeshLoader`、`TextureLoader` 直接从源文件创建运行时对象。 + +这导致: + +- 每次首次访问都重复导入 +- 无法形成 Library artifact +- 无法精确 reimport + +### 14.2 新的职责拆分 + +建议把当前体系拆成两类接口: + +#### IAssetImporter + +负责: + +- 从源文件导入到 artifact +- 生成 sub-asset +- 记录依赖 + +#### IArtifactLoader + +负责: + +- 从 artifact 还原为运行时资源对象 + +### 14.3 推荐接口草案 + +```text +IAssetImporter +- GetImporterName() +- GetImporterVersion() +- CanImport(sourcePath, meta) +- Import(request, output) +- GatherDependencies(request, outDeps) +- BuildDefaultMeta() + +IArtifactLoader +- GetResourceType() +- CanLoad(artifactObject) +- LoadFromArtifact(artifactObject, loadContext) +``` + +### 14.4 对现有类的映射 + +建议演进为: + +- `TextureLoader` 拆为 `TextureImporter + TextureArtifactLoader` +- `MeshLoader` 拆为 `ModelImporter + MeshArtifactLoader` +- `MaterialLoader` 拆为 `MaterialImporter + MaterialArtifactLoader` + +--- + +## 15. 各资产类型的初版实现细节 + +## 15.1 Texture 资产 + +### 15.1.1 输入 + +- `.png` +- `.jpg` +- `.jpeg` +- 后续再扩展其它格式 + +### 15.1.2 输出 + +建议输出: + +- `xctex` 二进制文件 +- 可选 `artifact.json` + +### 15.1.3 初版 `xctex` 内容 + +- magic +- schemaVersion +- textureFormat +- width +- height +- mipCount +- colorSpace +- importFlags +- payloadOffset +- payloadSize +- pixel/compressed data + +### 15.1.4 初版策略 + +初版先不做 GPU 压缩纹理缓存,先做: + +- RGBA8_UNORM +- 可选 sRGB 标记 +- 可选 mipmap 预生成 + +先把“只导一次、只读 artifact”做对,再谈 BC 压缩。 + +## 15.2 Material 资产 + +### 15.2.1 当前问题 + +当前 `Material` 运行时对象直接持有纹理 handle,容易在加载材质时把所有纹理一并拉进来。 + +### 15.2.2 重构目标 + +`Material` artifact 应该存: + +- shader 标识 +- render state +- float/int/vector 属性 +- 纹理槽位对应的 `AssetRef` + +而不是直接保存已加载的纹理对象。 + +### 15.2.3 运行时行为 + +运行时 `Material` 应支持: + +- 保存纹理引用 +- 延迟请求实际 `Texture` +- 由渲染管线按需解析纹理 + +这样可以避免打开 scene 时把所有材质贴图全同步加载。 + +## 15.3 Model 资产 + +### 15.3.1 当前问题 + +当前 `MeshLoader` 会: + +- 读取 `obj` +- 生成 mesh +- 导入材质 +- 导入贴图 +- 把材质和贴图一起挂到 `Mesh` + +这会造成过度加载。 + +### 15.3.2 新设计 + +`ModelImporter` 应该把模型导入成一个 artifact 包,内部可包含多个 sub-asset: + +- 主模型对象 +- 一个或多个 `Mesh` sub-asset +- 零个或多个默认 `Material` sub-asset +- 零个或多个内嵌纹理 sub-asset + +外部贴图不应再被复制进入 mesh runtime 对象。 + +### 15.3.3 外部贴图处理 + +如果 `obj/mtl` 引用的是外部贴图文件: + +- 先通过相对路径解析到 `Assets` 内真实资源 +- 找到该贴图的 `AssetGUID` +- 在导出的 `Material` sub-asset 中写入 `AssetRef` +- 不在 model artifact 中嵌入外部贴图像素数据 + +只有模型文件内嵌的纹理,才作为 model 的 sub-asset 输出。 + +### 15.3.4 `xcmesh` 内容建议 + +- header +- vertex buffer layout +- vertex count +- index count +- bounds +- sections +- 默认材质 `AssetRef` +- vertex data blob +- index data blob + +### 15.3.5 mesh runtime 对象要求 + +新的 `Mesh` 运行时对象应该: + +- 持有顶点、索引、section、bounds +- 可选保存默认材质引用 +- 不再持有 `Texture*` +- 不再默认拥有一堆已创建好的 `Material*` + +## 15.4 Scene 资产 + +### 15.4.1 这次不把 Scene 打成必须的二进制 artifact + +初版不要求 scene 一定导成二进制 artifact。 + +原因: + +- 当前性能瓶颈不在 scene 文本解析 +- 瓶颈在 scene 打开时触发原始模型和贴图导入 + +所以初版 scene 可以继续保留 `.xc` 文本格式,但必须改引用方式。 + +### 15.4.2 Scene 引用重构 + +scene 中不应再保存: + +- `mesh=Assets/Models/backpack/backpack.obj` + +而应改成: + +- `meshGuid=` +- `meshLocalID=` + +或者后续再统一为通用 `AssetRef` 文本格式。 + +### 15.4.3 兼容期 + +初版建议保留一段兼容窗口: + +- 读取优先使用 `guid + localID` +- 如果没有,再回退读取旧路径字段 +- 迁移完成后,写出只写新格式 + +--- + +## 16. 组件层重构要求 + +### 16.1 MeshFilterComponent + +当前问题: + +- 反序列化时立刻同步 `Load()` + +新要求: + +- 反序列化只恢复 `AssetRef` +- 不立即导入或加载 +- `GetMesh()` 可做 lazy resolve +- editor 可根据需要同步或异步请求资源 + +### 16.2 MeshRendererComponent + +当前问题: + +- 材质路径字符串 + 反序列化时直接同步加载 + +新要求: + +- 保存 `AssetRef` 列表 +- 反序列化只恢复引用,不立即创建资源 +- 渲染时按需 resolve + +### 16.3 为什么这一步必须做 + +如果不改组件层,即使有 `Library`: + +- scene 打开时仍会同步加载所有资源 +- 只是“从 artifact 同步加载”,但主线程仍会卡 + +所以这次重构不只是 cache 落盘,还必须把组件反序列化改成纯数据恢复。 + +--- + +## 17. Runtime 资源加载重构 + +### 17.1 ResourceManager 的新职责 + +重构后 `ResourceManager` 应只负责: + +- 运行时对象 cache +- artifact object 到 runtime object 的映射 +- 引用计数 +- 卸载策略 + +不再直接负责: + +- 解析 `.obj` +- 解码 `.png/.jpg` +- 决定 asset 是否需要 reimport + +### 17.2 AssetDatabase 与 ResourceManager 的关系 + +建议关系: + +- `AssetDatabase` 负责编辑器世界的资产身份与 artifact 定位 +- `ResourceManager` 负责运行时对象生命周期 + +流程: + +1. Component 持有 `AssetRef` +2. 请求资源时交给 `AssetDatabase` +3. `AssetDatabase` 找到当前有效 artifact object +4. `ResourceManager` 用 `IArtifactLoader` 加载 artifact +5. 缓存 runtime object + +### 17.3 渲染时的按需纹理解析 + +例如 forward pipeline 只需要 `baseColorTexture` 时: + +- 只解析该槽位对应的 `AssetRef` +- 不应自动加载 normal/specular/occlusion + +这一步是解决“模型材质导入过重”的关键。 + +--- + +## 18. 依赖系统设计 + +### 18.1 为什么一定要有依赖系统 + +没有依赖系统,就无法正确回答: + +- `obj` 改了要不要重导? +- `mtl` 改了要不要重导? +- 外部贴图改了要不要重导? +- importer 设置改了要不要重导? +- 导入器版本升级了要不要重导? + +### 18.2 依赖类型 + +初版至少支持: + +- 主源文件依赖 +- 侧车文件依赖 +- 资产依赖 +- importer 版本依赖 + +### 18.3 以 OBJ 为例 + +一个 `backpack.obj` 的导入依赖可能包含: + +- `backpack.obj` +- `backpack.mtl` +- 由 `.mtl` 解析出的外部贴图路径 +- `ModelImporter` 的导入设置 hash +- `ModelImporter` 版本号 + +### 18.4 失效规则 + +以下任一变化应触发 reimport: + +- 源文件内容 hash 变化 +- `.meta` 设置变化 +- importer version 变化 +- 依赖文件内容变化 +- 依赖资产引用变化 +- artifact 文件缺失 +- artifact schema version 不匹配 + +--- + +## 19. 异步导入与异步加载设计 + +### 19.1 当前问题 + +当前 `AsyncLoader` 不是真正的后台导入系统。 + +### 19.2 新的任务模型 + +建议分成两类任务: + +#### Import Task + +负责: + +- 检查 artifact 是否失效 +- 读取源文件 +- 执行 importer +- 写 Library + +#### Runtime Load Task + +负责: + +- 从 artifact 读取 +- 创建运行时对象 + +### 19.3 初版落地策略 + +建议分两阶段: + +#### 第一阶段 + +- 先做正式持久化 Library +- artifact miss 时允许同步 reimport +- 目标是第二次打开同资源时显著加速 + +#### 第二阶段 + +- 引入后台 import 队列 +- scene 打开时只恢复引用并排队资源 +- 用 placeholder 和进度状态代替主线程长阻塞 + +这样风险更低,也更容易调试。 + +--- + +## 20. Editor 侧流程重构 + +### 20.1 打开项目 + +打开项目时应执行: + +1. 确保 `Assets/` 存在 +2. 确保 `Library/` 存在 +3. 扫描 `Assets/` +4. 自动补齐缺失 `.meta` +5. 构建或修复 `SourceAssetDB` +6. 校验 `ArtifactDB` +7. 清理明显损坏或孤立的 artifact 记录 + +### 20.2 新增/导入文件 + +新文件出现时: + +- 自动生成 `.meta` +- 分配 `AssetGUID` +- 进入 `SourceAssetDB` +- 标记为 `NeedsImport` + +### 20.3 重命名/移动文件 + +在 editor 内部重命名/移动时: + +- 文件与 `.meta` 一起移动 +- `AssetGUID` 保持不变 +- 只更新路径索引 +- scene 不需要改引用 + +这是新系统最核心的收益之一。 + +### 20.4 删除文件 + +删除文件时: + +- 从 `SourceAssetDB` 标记删除 +- `ArtifactDB` 中对应 artifact 标记为 orphan +- 后台或手动 GC 清理 artifact 文件 + +### 20.5 Reimport + +需要支持: + +- Reimport 当前资产 +- Reimport All +- 查看导入日志 +- 查看当前 artifact key +- 查看依赖列表 + +--- + +## 21. 场景与序列化迁移方案 + +### 21.1 目标 + +把 scene 与 prefab 从路径引用迁移到 `AssetRef`。 + +### 21.2 迁移策略 + +推荐采用三阶段迁移: + +#### 阶段 A:双读 + +- 读新格式 +- 读不到则回退旧路径格式 + +#### 阶段 B:双写 + +- 新保存写 `guid + localID` +- 可选保留 `pathFallback` + +#### 阶段 C:单写 + +- 正式只写新格式 +- 旧格式仅保留读兼容 + +### 21.3 迁移工具 + +需要提供一个项目级迁移工具: + +- 扫描所有 scene / prefab / material +- 尝试把路径解析成 `AssetGUID` +- 找不到则报错并生成报告 +- 成功后回写新引用 + +--- + +## 22. 对当前 Backpack 问题的直接收益 + +以 `backpack.obj` 为例,重构后流程将变为: + +### 第一次导入 + +- `ModelImporter` 读取 `backpack.obj` +- 解析 `backpack.mtl` +- 为模型生成 `Mesh` artifact +- 为默认材质生成 `Material` sub-asset +- 外部贴图只写成 `AssetRef` +- 贴图由各自的 `TextureImporter` 生成单独 artifact +- 导入结果写入 `Library/Artifacts` + +### 之后打开 scene + +- scene 只恢复 `AssetRef` +- editor 查 `ArtifactDB` +- 直接从 `xcmesh` 读取 mesh +- 材质只恢复引用 +- forward pipeline 只在需要时解析 `baseColorTexture` +- 不再重新解析原始 `obj` +- 不再重新解码所有原始 `jpg/png` + +也就是说: + +- 首次导入仍然可能慢 +- 但第二次及以后,scene 打开不应再把 OBJ/贴图导入当场重做 + +--- + +## 23. 推荐的模块拆分 + +### 23.1 Engine 层新增/调整 + +建议新增或调整以下模块: + +```text +engine/include/XCEngine/Core/Asset/ +├── AssetGUID.h +├── AssetRef.h +├── ArtifactKey.h +├── ArtifactManifest.h +├── IArtifactLoader.h +└── RuntimeAssetResolver.h +``` + +```text +engine/src/Core/Asset/ +├── AssetGUID.cpp +├── AssetRef.cpp +├── ArtifactKey.cpp +├── ArtifactManifest.cpp +└── RuntimeAssetResolver.cpp +``` + +### 23.2 Editor 层新增模块 + +建议新增: + +```text +editor/src/AssetPipeline/ +├── AssetMetaSerializer.h/.cpp +├── SourceAssetDatabase.h/.cpp +├── ArtifactDatabase.h/.cpp +├── ArtifactStore.h/.cpp +├── AssetImporterRegistry.h/.cpp +├── TextureImporter.h/.cpp +├── ModelImporter.h/.cpp +├── MaterialImporter.h/.cpp +├── AssetDependencyScanner.h/.cpp +├── ImportScheduler.h/.cpp +├── ImportLogger.h/.cpp +└── ProjectAssetScanner.h/.cpp +``` + +### 23.3 对现有模块的调整 + +- `ResourceManager` 从“源文件加载器入口”改为“artifact runtime cache” +- `MeshLoader` 拆分 +- `TextureLoader` 拆分 +- `MaterialLoader` 拆分 +- `MeshFilterComponent` 改为存 `AssetRef` +- `MeshRendererComponent` 改为存 `AssetRef` +- `ProjectManager` 扫描时支持 `.meta` +- `Scene` 序列化升级版本 + +--- + +## 24. 分阶段实施计划 + +## 阶段 0:基础类型与工程边界收口 + +目标: + +- 引入 `AssetGUID` +- 引入 `AssetRef` +- 把“路径 hash 充当资产身份”的设计正式停止 + +产出: + +- `AssetGUID` 类型 +- `AssetRef` 类型 +- `ResourceGUID` 与 `AssetGUID` 的职责拆分 + +当前状态: + +- 已完成。 +- 已新增 `AssetGUID` / `AssetRef` 基础类型,并开始接入运行时加载链路。 + +## 阶段 1:`.meta` 与 SourceAssetDB + +目标: + +- 为 `Assets` 中所有文件/文件夹补齐 `.meta` +- 建立 `guid/path/meta/importer` 索引 + +产出: + +- `.meta` 读写器 +- 资产扫描器 +- `SourceAssetDB` +- 资产移动/重命名时 GUID 保持不变 + +当前状态: + +- 已完成。 +- 已实现 `.meta` 自动生成、读取、写回与资产扫描。 +- 已在 `ProjectManager` 中处理 `.meta` sidecar 的删除、移动、重命名与隐藏显示。 + +## 阶段 2:Artifact Store 与 ArtifactDB + +目标: + +- 把导入结果正式落盘进 `Library/Artifacts` +- 用 `ArtifactDB` 记录导入产物和依赖 + +产出: + +- `ArtifactKey` +- `ArtifactDB` +- `ArtifactStore` +- 依赖与失效判断 + +当前状态: + +- 已完成。 +- 已落地 `Library/SourceAssetDB`、`Library/ArtifactDB`、`Library/Artifacts` 三层结构。 +- 已实现基于 `sourceHash/metaHash/importerVersion/size/writeTime` 的失效判断。 + +## 阶段 3:TextureImporter + +目标: + +- 纹理先走通从源文件到 artifact 的闭环 + +产出: + +- `TextureImporter` +- `TextureArtifactLoader` +- `xctex` 格式 + +当前状态: + +- 已完成。 +- 当前纹理资源首次导入时会生成 `xctex`,后续优先从 artifact 读取。 + +## 阶段 4:ModelImporter + +目标: + +- 模型导入改为 artifact 生成,不再 runtime 直接读源 `obj` + +产出: + +- `ModelImporter` +- `xcmesh` 格式 +- 默认材质 sub-asset +- 外部贴图改成 `AssetRef` + +当前状态: + +- 已完成初版。 +- 当前模型首次导入时会生成 `main.xcmesh` 与关联 `texture_N.xctex`。 +- 仍未完成“外部贴图全局去重 + 材质独立 artifact + 真正按引用延迟加载”的最终形态,因此这里只能算阶段 4 初版完成。 + +## 阶段 5:Material 系统 lazy 引用化 + +目标: + +- 材质不再一加载就拉起所有纹理 + +产出: + +- `xcmat` +- 纹理引用延迟解析 +- `Material` runtime 结构调整 + +当前状态: + +- 部分完成。 +- 已补齐材质 artifact 读写所需的 tag/property/texture binding 访问接口。 +- `xcmat` 尚未落地,贴图仍是 eager load,不是最终目标状态。 + +## 阶段 6:Scene / Component 引用迁移 + +目标: + +- 路径引用改成 `guid + localID` + +产出: + +- 新 scene 序列化格式 +- 旧路径兼容读取 +- 迁移工具 + +当前状态: + +- 部分完成。 +- `MeshFilterComponent` 与 `MeshRendererComponent` 已双写 `path + AssetRef`,并保留旧数据兼容。 +- Scene 全量序列化升级、统一迁移工具、其他组件资产引用迁移尚未完成。 + +## 阶段 7:异步导入与异步加载 + +目标: + +- 打开 scene 主线程只恢复引用,不做大规模阻塞导入 + +产出: + +- import task 队列 +- runtime load task 队列 +- placeholder 与状态提示 + +当前状态: + +- 未开始。 +- 这是当前 editor 仍会被大 `obj` 场景卡住的首要剩余任务。 + +## 阶段 8:清理、GC、工具与可视化 + +目标: + +- 系统可维护、可调试、可清理 + +产出: + +- orphan artifact 清理 +- Reimport All +- 依赖图查看 +- 导入日志面板 + +当前状态: + +- 未开始。 + +--- + +## 25. 测试与验证方案 + +### 25.1 单元测试 + +必须补这些测试: + +- `.meta` 缺失时自动生成 GUID +- 重命名/移动后 GUID 保持不变 +- 同一资产不同 importer 设置产生不同 artifact key +- source 改变时 artifact 失效 +- 依赖贴图变化时 material 或 model 重新判定 +- sub-asset localID 稳定性 +- scene 新旧格式兼容读取 + +### 25.2 集成测试 + +必须补这些流程测试: + +- 首次打开 backpack scene 触发导入 +- 第二次打开 backpack scene 不再读取原始 `obj` +- 仅改 `diffuse.jpg` 时,不重新导入无关资产 +- 移动模型文件后 scene 仍能打开 +- 删除 artifact 后系统能够自动重建 + +### 25.3 性能验证 + +必须记录的指标: + +- 首次导入耗时 +- warm 打开 scene 耗时 +- 主线程阻塞时间 +- texture 解码时间 +- model import 时间 +- GPU 首帧上传时间 + +建议加入正式 instrumentation: + +- `Scene open` +- `Asset resolve` +- `Import task` +- `Artifact load` +- `GPU upload` + +--- + +## 26. 风险与注意事项 + +### 26.1 不能再把路径当 GUID + +这是本次重构的底线。如果不先解决这个问题,后面所有缓存都会在重命名/移动后变脆。 + +### 26.2 sub-asset LocalID 稳定性很重要 + +如果 `obj` 重导后 mesh localID 改了,即使 `AssetGUID` 不变,scene 里的引用仍然会断。 + +因此 localID 的生成规则必须: + +- 基于稳定逻辑路径 +- 不依赖运行时顺序 +- 在 importer 内版本化 + +### 26.3 初版不要急着做复杂压缩 + +真正需要优先解决的是: + +- 导入和加载分离 +- Library 持久化 +- scene 引用改成 GUID + +纹理压缩和高级优化可以后做。 + +### 26.4 当前 Preview 缓存可以暂不并入新 Library + +当前 editor 已经有 `.xceditor/thumbs` 预览缓存。 + +初版资产导入缓存系统可以先不动它,后续再考虑统一到 `Library/PreviewCache`。 + +### 26.5 需要预留 schema/version + +以下内容都必须版本化: + +- `.meta` format version +- artifact manifest version +- `xcmesh/xctex/xcmat` format version +- importer version +- project Library cache version + +否则后续升级会非常痛苦。 + +--- + +## 27. 最终预期结果 + +这次重构完成后,系统应达到以下效果: + +- 打开 scene 时,不再直接解析源 `obj/png/jpg` +- 资产改名或移动后,scene 引用仍然稳定 +- 首次导入慢,但只慢一次 +- 后续打开 scene 优先走 `Library` +- `Mesh`、`Material`、`Texture` 的职责边界清晰 +- editor 可以正式支持 Reimport、依赖跟踪、缓存清理 +- 后续扩展 prefab、动画、打包、后台导入时,不需要推倒重来 + +一句话总结: + +这次要做的不是“给现在的 `ResourceManager` 加缓存”,而是把 XCEngine 的资产系统重构成: + +```text +Assets + .meta + -> +SourceAssetDB + -> +Importer + -> +ArtifactDB + Artifacts + -> +Artifact Loader + -> +ResourceManager + -> +Scene / Renderer +``` + +只有这样,当前“打开 scene 时同步导入 obj 模型导致卡顿”的问题,才会被系统性解决,而不是被局部绕开。 diff --git a/editor/src/Managers/ProjectManager.cpp b/editor/src/Managers/ProjectManager.cpp index 927cb3cc..f96877a3 100644 --- a/editor/src/Managers/ProjectManager.cpp +++ b/editor/src/Managers/ProjectManager.cpp @@ -121,6 +121,27 @@ fs::path MakeCaseOnlyRenameTempPath(const fs::path& sourcePath) { } } +fs::path GetMetaSidecarPath(const fs::path& assetPath) { + return fs::path(assetPath.wstring() + L".meta"); +} + +bool RenamePathCaseAware(const fs::path& sourcePath, const fs::path& destPath); + +void MoveMetaSidecarIfPresent(const fs::path& sourcePath, const fs::path& destPath) { + const fs::path sourceMetaPath = GetMetaSidecarPath(sourcePath); + if (!fs::exists(sourceMetaPath)) { + return; + } + + const fs::path destMetaPath = GetMetaSidecarPath(destPath); + RenamePathCaseAware(sourceMetaPath, destMetaPath); +} + +void RemoveMetaSidecarIfPresent(const fs::path& assetPath) { + std::error_code ec; + fs::remove_all(GetMetaSidecarPath(assetPath), ec); +} + bool RenamePathCaseAware(const fs::path& sourcePath, const fs::path& destPath) { if (MakePathKey(sourcePath) != MakePathKey(destPath)) { if (fs::exists(destPath)) { @@ -383,6 +404,7 @@ bool ProjectManager::DeleteItem(const std::string& fullPath) { } fs::remove_all(itemPath); + RemoveMetaSidecarIfPresent(itemPath); if (m_selectedItemPath == fullPath) { ClearSelection(); } @@ -422,6 +444,7 @@ bool ProjectManager::MoveItem(const std::string& sourceFullPath, const std::stri } fs::rename(sourcePath, destPath); + MoveMetaSidecarIfPresent(sourcePath, destPath); RefreshCurrentFolder(); return true; } catch (...) { @@ -458,6 +481,7 @@ bool ProjectManager::RenameItem(const std::string& sourceFullPath, const std::st if (!RenamePathCaseAware(sourcePath, destPath)) { return false; } + MoveMetaSidecarIfPresent(sourcePath, destPath); if (!m_selectedItemPath.empty() && MakePathKey(Utf8PathToWstring(m_selectedItemPath)) == MakePathKey(sourcePath)) @@ -501,6 +525,13 @@ AssetItemPtr ProjectManager::ScanDirectory(const std::wstring& path) { try { for (const auto& entry : fs::directory_iterator(path)) { std::wstring nameW = entry.path().filename().wstring(); + if (!entry.is_directory()) { + std::wstring lowerName = nameW; + std::transform(lowerName.begin(), lowerName.end(), lowerName.begin(), ::towlower); + if (lowerName.size() >= 5 && lowerName.substr(lowerName.size() - 5) == L".meta") { + continue; + } + } bool isFolder = entry.is_directory(); items.push_back(CreateAssetItem(entry.path().wstring(), nameW, isFolder)); } diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index d833b6eb..0cd7ac7e 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -242,6 +242,10 @@ add_library(XCEngine STATIC # Core/Asset (Resource System Core) ${CMAKE_CURRENT_SOURCE_DIR}/include/XCEngine/Core/Asset/IResource.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/XCEngine/Core/Asset/AssetGUID.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/XCEngine/Core/Asset/AssetRef.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/XCEngine/Core/Asset/ArtifactFormats.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/XCEngine/Core/Asset/AssetDatabase.h ${CMAKE_CURRENT_SOURCE_DIR}/include/XCEngine/Core/Asset/ResourceTypes.h ${CMAKE_CURRENT_SOURCE_DIR}/include/XCEngine/Core/Asset/ImportSettings.h ${CMAKE_CURRENT_SOURCE_DIR}/include/XCEngine/Core/Asset/ResourceHandle.h @@ -249,6 +253,8 @@ add_library(XCEngine STATIC ${CMAKE_CURRENT_SOURCE_DIR}/include/XCEngine/Core/Asset/ResourceCache.h ${CMAKE_CURRENT_SOURCE_DIR}/include/XCEngine/Core/Asset/AsyncLoader.h ${CMAKE_CURRENT_SOURCE_DIR}/include/XCEngine/Core/Asset/ResourceDependencyGraph.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/Core/Asset/AssetGUID.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/Core/Asset/AssetDatabase.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Core/Asset/ResourceManager.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Core/Asset/ResourceCache.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Core/Asset/AsyncLoader.cpp diff --git a/engine/include/XCEngine/Components/MeshFilterComponent.h b/engine/include/XCEngine/Components/MeshFilterComponent.h index d0ad61eb..9e2dfeca 100644 --- a/engine/include/XCEngine/Components/MeshFilterComponent.h +++ b/engine/include/XCEngine/Components/MeshFilterComponent.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -16,6 +17,7 @@ public: Resources::Mesh* GetMesh() const { return m_mesh.Get(); } const Resources::ResourceHandle& GetMeshHandle() const { return m_mesh; } const std::string& GetMeshPath() const { return m_meshPath; } + const Resources::AssetRef& GetMeshAssetRef() const { return m_meshRef; } void SetMeshPath(const std::string& meshPath); void SetMesh(const Resources::ResourceHandle& mesh); @@ -28,6 +30,7 @@ public: private: Resources::ResourceHandle m_mesh; std::string m_meshPath; + Resources::AssetRef m_meshRef; }; } // namespace Components diff --git a/engine/include/XCEngine/Components/MeshRendererComponent.h b/engine/include/XCEngine/Components/MeshRendererComponent.h index 5a3818e1..002241a3 100644 --- a/engine/include/XCEngine/Components/MeshRendererComponent.h +++ b/engine/include/XCEngine/Components/MeshRendererComponent.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -20,6 +21,7 @@ public: const Resources::ResourceHandle& GetMaterialHandle(size_t index) const; const std::string& GetMaterialPath(size_t index) const; const std::vector& GetMaterialPaths() const { return m_materialPaths; } + const std::vector& GetMaterialAssetRefs() const { return m_materialRefs; } void SetMaterialPath(size_t index, const std::string& materialPath); void SetMaterial(size_t index, const Resources::ResourceHandle& material); @@ -45,6 +47,7 @@ private: std::vector> m_materials; std::vector m_materialPaths; + std::vector m_materialRefs; bool m_castShadows = true; bool m_receiveShadows = true; uint32_t m_renderLayer = 0; diff --git a/engine/include/XCEngine/Core/Asset/ArtifactFormats.h b/engine/include/XCEngine/Core/Asset/ArtifactFormats.h new file mode 100644 index 00000000..32b5ca82 --- /dev/null +++ b/engine/include/XCEngine/Core/Asset/ArtifactFormats.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace XCEngine { +namespace Resources { + +constexpr Core::uint32 kTextureArtifactSchemaVersion = 1; +constexpr Core::uint32 kMeshArtifactSchemaVersion = 1; + +struct TextureArtifactHeader { + char magic[8] = { 'X', 'C', 'T', 'E', 'X', '0', '1', '\0' }; + Core::uint32 schemaVersion = kTextureArtifactSchemaVersion; + Core::uint32 textureType = 0; + Core::uint32 textureFormat = 0; + Core::uint32 width = 0; + Core::uint32 height = 0; + Core::uint32 depth = 0; + Core::uint32 mipLevels = 0; + Core::uint32 arraySize = 0; + Core::uint64 pixelDataSize = 0; +}; + +struct MeshArtifactHeader { + char magic[8] = { 'X', 'C', 'M', 'E', 'S', 'H', '1', '\0' }; + Core::uint32 schemaVersion = kMeshArtifactSchemaVersion; + Core::uint32 vertexCount = 0; + Core::uint32 vertexStride = 0; + Core::uint32 vertexAttributes = 0; + Core::uint32 indexCount = 0; + Core::uint32 use32BitIndex = 0; + Core::uint32 sectionCount = 0; + Core::uint32 materialCount = 0; + Core::uint32 textureCount = 0; + Math::Vector3 boundsMin = Math::Vector3::Zero(); + Math::Vector3 boundsMax = Math::Vector3::Zero(); + Core::uint64 vertexDataSize = 0; + Core::uint64 indexDataSize = 0; +}; + +struct MaterialArtifactHeader { + Core::int32 renderQueue = static_cast(MaterialRenderQueue::Geometry); + MaterialRenderState renderState = {}; + Core::uint32 tagCount = 0; + Core::uint32 propertyCount = 0; + Core::uint32 textureBindingCount = 0; +}; + +struct MaterialPropertyArtifact { + Core::uint32 propertyType = 0; + MaterialProperty::Value value = {}; +}; + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/include/XCEngine/Core/Asset/AssetDatabase.h b/engine/include/XCEngine/Core/Asset/AssetDatabase.h new file mode 100644 index 00000000..e4628f92 --- /dev/null +++ b/engine/include/XCEngine/Core/Asset/AssetDatabase.h @@ -0,0 +1,130 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace XCEngine { +namespace Resources { + +class AssetDatabase { +public: + struct SourceAssetRecord { + AssetGUID guid; + Containers::String relativePath; + Containers::String metaPath; + bool isFolder = false; + Containers::String importerName; + Core::uint32 importerVersion = 0; + Containers::String metaHash; + Containers::String sourceHash; + Core::uint64 sourceFileSize = 0; + Core::uint64 sourceWriteTime = 0; + Containers::String lastKnownArtifactKey; + }; + + struct ArtifactRecord { + Containers::String artifactKey; + AssetGUID assetGuid; + Containers::String importerName; + Core::uint32 importerVersion = 0; + ResourceType resourceType = ResourceType::Unknown; + Containers::String artifactDirectory; + Containers::String mainArtifactPath; + Containers::String sourceHash; + Containers::String metaHash; + Core::uint64 sourceFileSize = 0; + Core::uint64 sourceWriteTime = 0; + LocalID mainLocalID = kMainAssetLocalID; + }; + + struct ResolvedAsset { + bool exists = false; + bool artifactReady = false; + Containers::String absolutePath; + Containers::String relativePath; + AssetGUID assetGuid; + ResourceType resourceType = ResourceType::Unknown; + Containers::String artifactMainPath; + Containers::String artifactDirectory; + LocalID mainLocalID = kMainAssetLocalID; + }; + + void Initialize(const Containers::String& projectRoot); + void Shutdown(); + void Refresh(); + + bool ResolvePath(const Containers::String& requestPath, + Containers::String& outAbsolutePath, + Containers::String& outRelativePath) const; + bool TryGetAssetGuid(const Containers::String& requestPath, AssetGUID& outGuid) const; + bool TryGetAssetRef(const Containers::String& requestPath, ResourceType resourceType, AssetRef& outRef) const; + bool EnsureArtifact(const Containers::String& requestPath, + ResourceType requestedType, + ResolvedAsset& outAsset); + bool TryGetPrimaryAssetPath(const AssetGUID& guid, Containers::String& outRelativePath) const; + + const Containers::String& GetProjectRoot() const { return m_projectRoot; } + const Containers::String& GetAssetsRoot() const { return m_assetsRoot; } + const Containers::String& GetLibraryRoot() const { return m_libraryRoot; } + +private: + static constexpr Core::uint32 kCurrentImporterVersion = 1; + + void EnsureProjectLayout(); + void LoadSourceAssetDB(); + void SaveSourceAssetDB() const; + void LoadArtifactDB(); + void SaveArtifactDB() const; + void ScanAssets(); + void ScanAssetPath(const std::filesystem::path& path, + std::unordered_map& seenPaths); + void RemoveMissingRecords(const std::unordered_map& seenPaths); + + bool EnsureMetaForPath(const std::filesystem::path& sourcePath, + bool isFolder, + SourceAssetRecord& outRecord); + bool ReadMetaFile(const std::filesystem::path& metaPath, + SourceAssetRecord& inOutRecord) const; + void WriteMetaFile(const std::filesystem::path& metaPath, + const SourceAssetRecord& record) const; + + Containers::String NormalizeRelativePath(const std::filesystem::path& sourcePath) const; + static Containers::String NormalizePathString(const std::filesystem::path& path); + static Containers::String NormalizePathString(const Containers::String& path); + static Containers::String MakeKey(const Containers::String& path); + static Containers::String GetImporterNameForPath(const Containers::String& relativePath, bool isFolder); + static ResourceType GetPrimaryResourceTypeForImporter(const Containers::String& importerName); + + bool ShouldReimport(const SourceAssetRecord& sourceRecord, + const ArtifactRecord* artifactRecord) const; + bool ImportAsset(const SourceAssetRecord& sourceRecord, + ArtifactRecord& outRecord); + bool ImportTextureAsset(const SourceAssetRecord& sourceRecord, + ArtifactRecord& outRecord); + bool ImportModelAsset(const SourceAssetRecord& sourceRecord, + ArtifactRecord& outRecord); + + Containers::String BuildArtifactKey(const SourceAssetRecord& sourceRecord) const; + Containers::String BuildArtifactDirectory(const Containers::String& artifactKey) const; + static Containers::String ReadWholeFileText(const std::filesystem::path& path); + static Containers::String ComputeFileHash(const std::filesystem::path& path); + static Core::uint64 GetFileSizeValue(const std::filesystem::path& path); + static Core::uint64 GetFileWriteTimeValue(const std::filesystem::path& path); + + Containers::String m_projectRoot; + Containers::String m_assetsRoot; + Containers::String m_libraryRoot; + Containers::String m_sourceDbPath; + Containers::String m_artifactDbPath; + + std::unordered_map m_sourcesByPathKey; + std::unordered_map m_sourcesByGuid; + std::unordered_map m_artifactsByGuid; +}; + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/include/XCEngine/Core/Asset/AssetGUID.h b/engine/include/XCEngine/Core/Asset/AssetGUID.h new file mode 100644 index 00000000..92385b05 --- /dev/null +++ b/engine/include/XCEngine/Core/Asset/AssetGUID.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include + +namespace XCEngine { +namespace Resources { + +struct AssetGUID { + Core::uint64 high = 0; + Core::uint64 low = 0; + + AssetGUID() = default; + AssetGUID(Core::uint64 inHigh, Core::uint64 inLow) + : high(inHigh), low(inLow) {} + + bool IsValid() const { + return high != 0 || low != 0; + } + + bool operator==(const AssetGUID& other) const { + return high == other.high && low == other.low; + } + + bool operator!=(const AssetGUID& other) const { + return !(*this == other); + } + + static AssetGUID Generate(); + static bool TryParse(const Containers::String& text, AssetGUID& outGuid); + static AssetGUID ParseOrDefault(const Containers::String& text); + + Containers::String ToString() const; +}; + +using LocalID = Core::uint64; + +constexpr LocalID kInvalidLocalID = 0; +constexpr LocalID kMainAssetLocalID = 1; + +AssetGUID HashBytesToAssetGUID(const void* data, size_t size); +AssetGUID HashStringToAssetGUID(const Containers::String& text); + +} // namespace Resources +} // namespace XCEngine + +namespace std { +template<> +struct hash { + size_t operator()(const XCEngine::Resources::AssetGUID& guid) const noexcept { + return static_cast(guid.high ^ (guid.low * 0x9e3779b97f4a7c15ULL)); + } +}; +} diff --git a/engine/include/XCEngine/Core/Asset/AssetRef.h b/engine/include/XCEngine/Core/Asset/AssetRef.h new file mode 100644 index 00000000..e7cc4abe --- /dev/null +++ b/engine/include/XCEngine/Core/Asset/AssetRef.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include + +#include + +namespace XCEngine { +namespace Resources { + +struct AssetRef { + AssetGUID assetGuid; + LocalID localID = kInvalidLocalID; + ResourceType resourceType = ResourceType::Unknown; + + bool IsValid() const { + return assetGuid.IsValid() && localID != kInvalidLocalID && resourceType != ResourceType::Unknown; + } + + void Reset() { + assetGuid = AssetGUID(); + localID = kInvalidLocalID; + resourceType = ResourceType::Unknown; + } + + Containers::String ToString() const { + if (!IsValid()) { + return Containers::String(); + } + + return assetGuid.ToString() + ":" + Containers::String(std::to_string(localID).c_str()) + + ":" + Containers::String(GetResourceTypeName(resourceType)); + } +}; + +template +AssetRef MakeMainAssetRef(const AssetGUID& assetGuid) { + AssetRef ref; + ref.assetGuid = assetGuid; + ref.localID = kMainAssetLocalID; + ref.resourceType = GetResourceType(); + return ref; +} + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/include/XCEngine/Core/Asset/ResourceManager.h b/engine/include/XCEngine/Core/Asset/ResourceManager.h index 19c3b38b..44b132c3 100644 --- a/engine/include/XCEngine/Core/Asset/ResourceManager.h +++ b/engine/include/XCEngine/Core/Asset/ResourceManager.h @@ -1,6 +1,7 @@ #pragma once #include +#include "AssetDatabase.h" #include "ResourceCache.h" #include "AsyncLoader.h" #include "ResourceHandle.h" @@ -28,33 +29,26 @@ public: template ResourceHandle Load(const Containers::String& path, ImportSettings* settings = nullptr) { static_assert(std::is_base_of_v, "T must derive from IResource"); - - ResourceGUID guid = ResourceGUID::Generate(path); - - IResource* cached = FindInCache(guid); - if (cached) { - return ResourceHandle(static_cast(cached)); - } - - IResourceLoader* loader = FindLoader(GetResourceType()); - if (!loader) { - Debug::Logger::Get().Warning(Debug::LogCategory::FileSystem, - Containers::String("No loader found for resource type: ") + - GetResourceTypeName(GetResourceType())); + + LoadResult result = LoadResource(path, GetResourceType(), settings); + if (!result || result.resource == nullptr) { return ResourceHandle(); } - - LoadResult result = loader->Load(path, settings); - if (!result) { - Debug::Logger::Get().Error(Debug::LogCategory::FileSystem, - Containers::String("Failed to load resource: ") + path + " - " + result.errorMessage); - return ResourceHandle(); - } - - AddToCache(guid, result.resource); - + return ResourceHandle(static_cast(result.resource)); } + + template + ResourceHandle Load(const AssetRef& assetRef, ImportSettings* settings = nullptr) { + static_assert(std::is_base_of_v, "T must derive from IResource"); + + Containers::String path; + if (!assetRef.IsValid() || !m_assetDatabase.TryGetPrimaryAssetPath(assetRef.assetGuid, path)) { + return ResourceHandle(); + } + + return Load(path, settings); + } void LoadAsync(const Containers::String& path, ResourceType type, std::function callback); @@ -102,6 +96,8 @@ public: Containers::Array GetResourcePaths() const; void UnloadGroup(const Containers::Array& guids); + void RefreshAssetDatabase(); + bool TryGetAssetRef(const Containers::String& path, ResourceType resourceType, AssetRef& outRef) const; private: ResourceManager() = default; @@ -111,6 +107,7 @@ private: void AddToCache(ResourceGUID guid, IResource* resource); IResourceLoader* FindLoader(ResourceType type); void ReloadResource(ResourceGUID guid); + LoadResult LoadResource(const Containers::String& path, ResourceType type, ImportSettings* settings); Containers::String m_resourceRoot; Containers::HashMap m_resourceCache; @@ -121,6 +118,7 @@ private: size_t m_memoryUsage = 0; size_t m_memoryBudget = 512 * 1024 * 1024; + AssetDatabase m_assetDatabase; ResourceCache m_cache; Core::UniqueRef m_asyncLoader; Threading::Mutex m_mutex; diff --git a/engine/include/XCEngine/Core/Containers/HashMap.h b/engine/include/XCEngine/Core/Containers/HashMap.h index 6ffc24bb..1a2885b4 100644 --- a/engine/include/XCEngine/Core/Containers/HashMap.h +++ b/engine/include/XCEngine/Core/Containers/HashMap.h @@ -38,6 +38,7 @@ public: bool Insert(Pair&& pair); bool Erase(const Key& key); void Clear(); + Array GetPairs() const; size_t Size() const { return m_size; } bool Empty() const { return m_size == 0; } @@ -274,6 +275,19 @@ void HashMap::Clear() { m_size = 0; } +template +Array::Pair> HashMap::GetPairs() const { + Array pairs; + pairs.Reserve(m_size); + for (size_t bucketIndex = 0; bucketIndex < m_buckets.Size(); ++bucketIndex) { + const Bucket& bucket = m_buckets[bucketIndex]; + for (size_t pairIndex = 0; pairIndex < bucket.pairs.Size(); ++pairIndex) { + pairs.PushBack(bucket.pairs[pairIndex]); + } + } + return pairs; +} + template size_t HashMap::GetBucketIndex(const Key& key) const { if (m_bucketCount == 0) { diff --git a/engine/include/XCEngine/Resources/BuiltinResources.h b/engine/include/XCEngine/Resources/BuiltinResources.h new file mode 100644 index 00000000..e7bdefe6 --- /dev/null +++ b/engine/include/XCEngine/Resources/BuiltinResources.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +namespace XCEngine { +namespace Resources { + +enum class BuiltinPrimitiveType { + Cube, + Sphere, + Capsule, + Cylinder, + Plane, + Quad +}; + +bool IsBuiltinResourcePath(const Containers::String& path); +bool IsBuiltinMeshPath(const Containers::String& path); +bool IsBuiltinMaterialPath(const Containers::String& path); +bool IsBuiltinTexturePath(const Containers::String& path); + +const char* GetBuiltinPrimitiveDisplayName(BuiltinPrimitiveType primitiveType); +Containers::String GetBuiltinPrimitiveMeshPath(BuiltinPrimitiveType primitiveType); +Containers::String GetBuiltinDefaultPrimitiveMaterialPath(); +Containers::String GetBuiltinDefaultPrimitiveTexturePath(); + +bool TryParseBuiltinPrimitiveType(const Containers::String& path, BuiltinPrimitiveType& outPrimitiveType); + +LoadResult CreateBuiltinMeshResource(const Containers::String& path); +LoadResult CreateBuiltinMaterialResource(const Containers::String& path); +LoadResult CreateBuiltinTextureResource(const Containers::String& path); + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/include/XCEngine/Resources/Material/Material.h b/engine/include/XCEngine/Resources/Material/Material.h index b37dfd5d..42187fc0 100644 --- a/engine/include/XCEngine/Resources/Material/Material.h +++ b/engine/include/XCEngine/Resources/Material/Material.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace XCEngine { namespace Resources { @@ -126,6 +127,17 @@ struct MaterialProperty { MaterialProperty() : type(MaterialPropertyType::Float), refCount(0) {} }; +struct MaterialTagEntry { + Containers::String name; + Containers::String value; +}; + +struct MaterialTextureBinding { + Containers::String name; + Core::uint32 slot = 0; + ResourceHandle texture; +}; + class Material : public IResource { public: Material(); @@ -158,6 +170,9 @@ public: void RemoveTag(const Containers::String& name); void ClearTags(); Core::uint32 GetTagCount() const { return static_cast(m_tags.Size()); } + Containers::String GetTagName(Core::uint32 index) const; + Containers::String GetTagValue(Core::uint32 index) const; + const Containers::Array& GetTags() const { return m_tags; } void SetFloat(const Containers::String& name, float value); void SetFloat2(const Containers::String& name, const Math::Vector2& value); @@ -175,7 +190,11 @@ public: bool GetBool(const Containers::String& name) const; ResourceHandle GetTexture(const Containers::String& name) const; Core::uint32 GetTextureBindingCount() const { return static_cast(m_textureBindings.Size()); } - + Containers::String GetTextureBindingName(Core::uint32 index) const; + ResourceHandle GetTextureBindingTexture(Core::uint32 index) const; + const Containers::Array& GetTextureBindings() const { return m_textureBindings; } + std::vector GetProperties() const; + const Containers::Array& GetConstantBufferData() const { return m_constantBufferData; } void UpdateConstantBuffer(); void RecalculateMemorySize(); @@ -191,20 +210,10 @@ private: Core::int32 m_renderQueue = static_cast(MaterialRenderQueue::Geometry); MaterialRenderState m_renderState; Containers::String m_shaderPass; - struct TagEntry { - Containers::String name; - Containers::String value; - }; - Containers::Array m_tags; + Containers::Array m_tags; Containers::HashMap m_properties; Containers::Array m_constantBufferData; - - struct TextureBinding { - Containers::String name; - Core::uint32 slot; - ResourceHandle texture; - }; - Containers::Array m_textureBindings; + Containers::Array m_textureBindings; }; } // namespace Resources diff --git a/engine/include/XCEngine/Resources/Resources.h b/engine/include/XCEngine/Resources/Resources.h index 2d4bb6f0..5b8a95d2 100644 --- a/engine/include/XCEngine/Resources/Resources.h +++ b/engine/include/XCEngine/Resources/Resources.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/engine/src/Components/MeshFilterComponent.cpp b/engine/src/Components/MeshFilterComponent.cpp index 52d5b438..5abcaf39 100644 --- a/engine/src/Components/MeshFilterComponent.cpp +++ b/engine/src/Components/MeshFilterComponent.cpp @@ -13,21 +13,54 @@ std::string ToStdString(const Containers::String& value) { return std::string(value.CStr()); } +std::string EncodeAssetRef(const Resources::AssetRef& assetRef) { + if (!assetRef.IsValid()) { + return std::string(); + } + + return ToStdString(assetRef.assetGuid.ToString()) + "," + + std::to_string(assetRef.localID) + "," + + std::to_string(static_cast(assetRef.resourceType)); +} + +bool TryDecodeAssetRef(const std::string& value, Resources::AssetRef& outRef) { + const size_t firstComma = value.find(','); + const size_t secondComma = firstComma == std::string::npos ? std::string::npos : value.find(',', firstComma + 1); + if (firstComma == std::string::npos || secondComma == std::string::npos) { + return false; + } + + const Containers::String guidText(value.substr(0, firstComma).c_str()); + outRef.assetGuid = Resources::AssetGUID::ParseOrDefault(guidText); + outRef.localID = static_cast(std::stoull(value.substr(firstComma + 1, secondComma - firstComma - 1))); + outRef.resourceType = static_cast(std::stoi(value.substr(secondComma + 1))); + return outRef.IsValid(); +} + } // namespace void MeshFilterComponent::SetMeshPath(const std::string& meshPath) { m_meshPath = meshPath; if (m_meshPath.empty()) { m_mesh.Reset(); + m_meshRef.Reset(); return; } m_mesh = Resources::ResourceManager::Get().Load(m_meshPath.c_str()); + if (!Resources::ResourceManager::Get().TryGetAssetRef(m_meshPath.c_str(), Resources::ResourceType::Mesh, m_meshRef)) { + m_meshRef.Reset(); + } } void MeshFilterComponent::SetMesh(const Resources::ResourceHandle& mesh) { m_mesh = mesh; m_meshPath = mesh.Get() != nullptr ? ToStdString(mesh->GetPath()) : std::string(); + if (m_meshPath.empty()) { + m_meshRef.Reset(); + } else if (!Resources::ResourceManager::Get().TryGetAssetRef(m_meshPath.c_str(), Resources::ResourceType::Mesh, m_meshRef)) { + m_meshRef.Reset(); + } } void MeshFilterComponent::SetMesh(Resources::Mesh* mesh) { @@ -37,17 +70,22 @@ void MeshFilterComponent::SetMesh(Resources::Mesh* mesh) { void MeshFilterComponent::ClearMesh() { m_mesh.Reset(); m_meshPath.clear(); + m_meshRef.Reset(); } void MeshFilterComponent::Serialize(std::ostream& os) const { os << "mesh=" << m_meshPath << ";"; + os << "meshRef=" << EncodeAssetRef(m_meshRef) << ";"; } void MeshFilterComponent::Deserialize(std::istream& is) { m_mesh.Reset(); m_meshPath.clear(); + m_meshRef.Reset(); std::string token; + std::string pendingMeshPath; + Resources::AssetRef pendingMeshRef; while (std::getline(is, token, ';')) { if (token.empty()) { continue; @@ -62,9 +100,23 @@ void MeshFilterComponent::Deserialize(std::istream& is) { const std::string value = token.substr(eqPos + 1); if (key == "mesh") { - SetMeshPath(value); + pendingMeshPath = value; + } else if (key == "meshRef") { + TryDecodeAssetRef(value, pendingMeshRef); } } + + if (pendingMeshRef.IsValid()) { + m_meshRef = pendingMeshRef; + m_mesh = Resources::ResourceManager::Get().Load(pendingMeshRef); + if (m_mesh.Get() != nullptr) { + m_meshPath = ToStdString(m_mesh->GetPath()); + } else { + m_meshPath = pendingMeshPath; + } + } else if (!pendingMeshPath.empty()) { + SetMeshPath(pendingMeshPath); + } } } // namespace Components diff --git a/engine/src/Components/MeshRendererComponent.cpp b/engine/src/Components/MeshRendererComponent.cpp index 4c04ad62..b92c214d 100644 --- a/engine/src/Components/MeshRendererComponent.cpp +++ b/engine/src/Components/MeshRendererComponent.cpp @@ -13,6 +13,29 @@ std::string ToStdString(const Containers::String& value) { return std::string(value.CStr()); } +std::string EncodeAssetRef(const Resources::AssetRef& assetRef) { + if (!assetRef.IsValid()) { + return std::string(); + } + + return ToStdString(assetRef.assetGuid.ToString()) + "," + + std::to_string(assetRef.localID) + "," + + std::to_string(static_cast(assetRef.resourceType)); +} + +bool TryDecodeAssetRef(const std::string& value, Resources::AssetRef& outRef) { + const size_t firstComma = value.find(','); + const size_t secondComma = firstComma == std::string::npos ? std::string::npos : value.find(',', firstComma + 1); + if (firstComma == std::string::npos || secondComma == std::string::npos) { + return false; + } + + outRef.assetGuid = Resources::AssetGUID::ParseOrDefault(Containers::String(value.substr(0, firstComma).c_str())); + outRef.localID = static_cast(std::stoull(value.substr(firstComma + 1, secondComma - firstComma - 1))); + outRef.resourceType = static_cast(std::stoi(value.substr(secondComma + 1))); + return outRef.IsValid(); +} + std::vector SplitMaterialPaths(const std::string& value) { std::vector paths; if (value.empty()) { @@ -34,6 +57,32 @@ std::vector SplitMaterialPaths(const std::string& value) { return paths; } +std::vector SplitMaterialRefs(const std::string& value) { + std::vector refs; + if (value.empty()) { + return refs; + } + + size_t start = 0; + while (true) { + const size_t separator = value.find('|', start); + const std::string token = separator == std::string::npos + ? value.substr(start) + : value.substr(start, separator - start); + + Resources::AssetRef ref; + TryDecodeAssetRef(token, ref); + refs.push_back(ref); + + if (separator == std::string::npos) { + break; + } + start = separator + 1; + } + + return refs; +} + } // namespace Resources::Material* MeshRendererComponent::GetMaterial(size_t index) const { @@ -55,16 +104,24 @@ void MeshRendererComponent::SetMaterialPath(size_t index, const std::string& mat m_materialPaths[index] = materialPath; if (materialPath.empty()) { m_materials[index].Reset(); + m_materialRefs[index].Reset(); return; } m_materials[index] = Resources::ResourceManager::Get().Load(materialPath.c_str()); + if (!Resources::ResourceManager::Get().TryGetAssetRef(materialPath.c_str(), Resources::ResourceType::Material, m_materialRefs[index])) { + m_materialRefs[index].Reset(); + } } void MeshRendererComponent::SetMaterial(size_t index, const Resources::ResourceHandle& material) { EnsureMaterialSlot(index); m_materials[index] = material; m_materialPaths[index] = MaterialPathFromHandle(material); + if (m_materialPaths[index].empty() || + !Resources::ResourceManager::Get().TryGetAssetRef(m_materialPaths[index].c_str(), Resources::ResourceType::Material, m_materialRefs[index])) { + m_materialRefs[index].Reset(); + } } void MeshRendererComponent::SetMaterial(size_t index, Resources::Material* material) { @@ -74,14 +131,20 @@ void MeshRendererComponent::SetMaterial(size_t index, Resources::Material* mater void MeshRendererComponent::SetMaterials(const std::vector>& materials) { m_materials = materials; m_materialPaths.resize(materials.size()); + m_materialRefs.resize(materials.size()); for (size_t i = 0; i < materials.size(); ++i) { m_materialPaths[i] = MaterialPathFromHandle(materials[i]); + if (m_materialPaths[i].empty() || + !Resources::ResourceManager::Get().TryGetAssetRef(m_materialPaths[i].c_str(), Resources::ResourceType::Material, m_materialRefs[i])) { + m_materialRefs[i].Reset(); + } } } void MeshRendererComponent::ClearMaterials() { m_materials.clear(); m_materialPaths.clear(); + m_materialRefs.clear(); } void MeshRendererComponent::Serialize(std::ostream& os) const { @@ -93,6 +156,14 @@ void MeshRendererComponent::Serialize(std::ostream& os) const { os << m_materialPaths[i]; } os << ";"; + os << "materialRefs="; + for (size_t i = 0; i < m_materialRefs.size(); ++i) { + if (i > 0) { + os << "|"; + } + os << EncodeAssetRef(m_materialRefs[i]); + } + os << ";"; os << "castShadows=" << (m_castShadows ? 1 : 0) << ";"; os << "receiveShadows=" << (m_receiveShadows ? 1 : 0) << ";"; os << "renderLayer=" << m_renderLayer << ";"; @@ -105,6 +176,7 @@ void MeshRendererComponent::Deserialize(std::istream& is) { m_renderLayer = 0; std::string token; + std::vector pendingMaterialRefs; while (std::getline(is, token, ';')) { if (token.empty()) { continue; @@ -121,9 +193,9 @@ void MeshRendererComponent::Deserialize(std::istream& is) { if (key == "materials") { m_materialPaths = SplitMaterialPaths(value); m_materials.resize(m_materialPaths.size()); - for (size_t i = 0; i < m_materialPaths.size(); ++i) { - SetMaterialPath(i, m_materialPaths[i]); - } + m_materialRefs.resize(m_materialPaths.size()); + } else if (key == "materialRefs") { + pendingMaterialRefs = SplitMaterialRefs(value); } else if (key == "castShadows") { m_castShadows = (std::stoi(value) != 0); } else if (key == "receiveShadows") { @@ -132,12 +204,32 @@ void MeshRendererComponent::Deserialize(std::istream& is) { m_renderLayer = static_cast(std::stoul(value)); } } + + if (pendingMaterialRefs.size() > m_materialPaths.size()) { + m_materialPaths.resize(pendingMaterialRefs.size()); + m_materials.resize(pendingMaterialRefs.size()); + m_materialRefs.resize(pendingMaterialRefs.size()); + } + + for (size_t i = 0; i < m_materialPaths.size(); ++i) { + if (i < pendingMaterialRefs.size() && pendingMaterialRefs[i].IsValid()) { + m_materialRefs[i] = pendingMaterialRefs[i]; + m_materials[i] = Resources::ResourceManager::Get().Load(pendingMaterialRefs[i]); + if (m_materials[i].Get() != nullptr) { + m_materialPaths[i] = MaterialPathFromHandle(m_materials[i]); + continue; + } + } + + SetMaterialPath(i, m_materialPaths[i]); + } } void MeshRendererComponent::EnsureMaterialSlot(size_t index) { if (index >= m_materials.size()) { m_materials.resize(index + 1); m_materialPaths.resize(index + 1); + m_materialRefs.resize(index + 1); } } diff --git a/engine/src/Core/Asset/AssetDatabase.cpp b/engine/src/Core/Asset/AssetDatabase.cpp new file mode 100644 index 00000000..efc2c07a --- /dev/null +++ b/engine/src/Core/Asset/AssetDatabase.cpp @@ -0,0 +1,1089 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace XCEngine { +namespace Resources { + +namespace fs = std::filesystem; + +namespace { + +std::string ToStdString(const Containers::String& value) { + return std::string(value.CStr()); +} + +Containers::String ToContainersString(const std::string& value) { + return Containers::String(value.c_str()); +} + +std::string TrimCopy(const std::string& text) { + const auto begin = std::find_if_not(text.begin(), text.end(), [](unsigned char ch) { + return std::isspace(ch) != 0; + }); + if (begin == text.end()) { + return std::string(); + } + + const auto end = std::find_if_not(text.rbegin(), text.rend(), [](unsigned char ch) { + return std::isspace(ch) != 0; + }).base(); + return std::string(begin, end); +} + +std::string ToLowerCopy(std::string text) { + std::transform(text.begin(), text.end(), text.begin(), [](unsigned char ch) { + return static_cast(std::tolower(ch)); + }); + return text; +} + +std::string EscapeField(const std::string& value) { + std::string escaped; + escaped.reserve(value.size()); + for (const char ch : value) { + if (ch == '\\' || ch == '\t' || ch == '\n' || ch == '\r') { + escaped.push_back('\\'); + switch (ch) { + case '\t': + escaped.push_back('t'); + break; + case '\n': + escaped.push_back('n'); + break; + case '\r': + escaped.push_back('r'); + break; + default: + escaped.push_back(ch); + break; + } + } else { + escaped.push_back(ch); + } + } + return escaped; +} + +std::string UnescapeField(const std::string& value) { + std::string result; + result.reserve(value.size()); + for (size_t index = 0; index < value.size(); ++index) { + if (value[index] == '\\' && index + 1 < value.size()) { + ++index; + switch (value[index]) { + case 't': + result.push_back('\t'); + break; + case 'n': + result.push_back('\n'); + break; + case 'r': + result.push_back('\r'); + break; + default: + result.push_back(value[index]); + break; + } + } else { + result.push_back(value[index]); + } + } + return result; +} + +std::vector SplitFields(const std::string& line) { + std::vector fields; + std::string current; + bool escaping = false; + + for (const char ch : line) { + if (escaping) { + current.push_back('\\'); + current.push_back(ch); + escaping = false; + continue; + } + + if (ch == '\\') { + escaping = true; + continue; + } + + if (ch == '\t') { + fields.push_back(UnescapeField(current)); + current.clear(); + continue; + } + + current.push_back(ch); + } + + if (escaping) { + current.push_back('\\'); + } + fields.push_back(UnescapeField(current)); + return fields; +} + +void WriteString(std::ofstream& stream, const Containers::String& value) { + const Core::uint32 length = static_cast(value.Length()); + stream.write(reinterpret_cast(&length), sizeof(length)); + if (length > 0) { + stream.write(value.CStr(), length); + } +} + +Containers::String ReadString(std::ifstream& stream) { + Core::uint32 length = 0; + stream.read(reinterpret_cast(&length), sizeof(length)); + if (!stream || length == 0) { + return Containers::String(); + } + + std::string buffer(length, '\0'); + stream.read(buffer.data(), length); + if (!stream) { + return Containers::String(); + } + + return ToContainersString(buffer); +} + +bool WriteTextureArtifactFile(const fs::path& artifactPath, const Texture& texture) { + std::ofstream output(artifactPath, std::ios::binary | std::ios::trunc); + if (!output.is_open()) { + return false; + } + + TextureArtifactHeader header; + header.textureType = static_cast(texture.GetTextureType()); + header.textureFormat = static_cast(texture.GetFormat()); + header.width = texture.GetWidth(); + header.height = texture.GetHeight(); + header.depth = texture.GetDepth(); + header.mipLevels = texture.GetMipLevels(); + header.arraySize = texture.GetArraySize(); + header.pixelDataSize = static_cast(texture.GetPixelDataSize()); + + output.write(reinterpret_cast(&header), sizeof(header)); + if (texture.GetPixelDataSize() > 0) { + output.write(static_cast(texture.GetPixelData()), texture.GetPixelDataSize()); + } + + return static_cast(output); +} + +std::vector GatherMaterialProperties(const Material& material) { + return material.GetProperties(); +} + +void WriteMaterialBlock(std::ofstream& output, + const Material& material, + const std::unordered_map& textureFileNames) { + WriteString(output, material.GetName()); + WriteString(output, material.GetPath()); + WriteString(output, material.GetShaderPass()); + + MaterialArtifactHeader header; + header.renderQueue = material.GetRenderQueue(); + header.renderState = material.GetRenderState(); + header.tagCount = material.GetTagCount(); + + const std::vector properties = GatherMaterialProperties(material); + std::vector nonTextureProperties; + nonTextureProperties.reserve(properties.size()); + for (const MaterialProperty& property : properties) { + if (property.type == MaterialPropertyType::Texture || + property.type == MaterialPropertyType::Cubemap) { + continue; + } + nonTextureProperties.push_back(property); + } + + header.propertyCount = static_cast(nonTextureProperties.size()); + header.textureBindingCount = material.GetTextureBindingCount(); + output.write(reinterpret_cast(&header), sizeof(header)); + + for (Core::uint32 tagIndex = 0; tagIndex < material.GetTagCount(); ++tagIndex) { + WriteString(output, material.GetTagName(tagIndex)); + WriteString(output, material.GetTagValue(tagIndex)); + } + + for (const MaterialProperty& property : nonTextureProperties) { + WriteString(output, property.name); + + MaterialPropertyArtifact propertyArtifact; + propertyArtifact.propertyType = static_cast(property.type); + propertyArtifact.value = property.value; + output.write(reinterpret_cast(&propertyArtifact), sizeof(propertyArtifact)); + } + + for (Core::uint32 bindingIndex = 0; bindingIndex < material.GetTextureBindingCount(); ++bindingIndex) { + const Containers::String bindingName = material.GetTextureBindingName(bindingIndex); + const Texture* texture = material.GetTextureBindingTexture(bindingIndex).Get(); + auto fileIt = texture != nullptr ? textureFileNames.find(texture) : textureFileNames.end(); + + WriteString(output, bindingName); + WriteString(output, + fileIt != textureFileNames.end() + ? ToContainersString(fileIt->second) + : Containers::String()); + } +} + +bool WriteMeshArtifactFile(const fs::path& artifactPath, const Mesh& mesh) { + std::ofstream output(artifactPath, std::ios::binary | std::ios::trunc); + if (!output.is_open()) { + return false; + } + + MeshArtifactHeader header; + header.vertexCount = mesh.GetVertexCount(); + header.vertexStride = mesh.GetVertexStride(); + header.vertexAttributes = static_cast(mesh.GetVertexAttributes()); + header.indexCount = mesh.GetIndexCount(); + header.use32BitIndex = mesh.IsUse32BitIndex() ? 1u : 0u; + header.sectionCount = static_cast(mesh.GetSections().Size()); + header.materialCount = static_cast(mesh.GetMaterials().Size()); + header.textureCount = static_cast(mesh.GetTextures().Size()); + header.boundsMin = mesh.GetBounds().GetMin(); + header.boundsMax = mesh.GetBounds().GetMax(); + header.vertexDataSize = static_cast(mesh.GetVertexDataSize()); + header.indexDataSize = static_cast(mesh.GetIndexDataSize()); + + output.write(reinterpret_cast(&header), sizeof(header)); + for (const MeshSection& section : mesh.GetSections()) { + output.write(reinterpret_cast(§ion), sizeof(section)); + } + + if (mesh.GetVertexDataSize() > 0) { + output.write(static_cast(mesh.GetVertexData()), mesh.GetVertexDataSize()); + } + if (mesh.GetIndexDataSize() > 0) { + output.write(static_cast(mesh.GetIndexData()), mesh.GetIndexDataSize()); + } + + std::unordered_map textureFileNames; + for (size_t textureIndex = 0; textureIndex < mesh.GetTextures().Size(); ++textureIndex) { + const Texture* texture = mesh.GetTextures()[textureIndex]; + if (texture == nullptr) { + continue; + } + + const std::string fileName = "texture_" + std::to_string(textureIndex) + ".xctex"; + textureFileNames.emplace(texture, fileName); + WriteString(output, ToContainersString(fileName)); + } + + for (size_t materialIndex = 0; materialIndex < mesh.GetMaterials().Size(); ++materialIndex) { + const Material* material = mesh.GetMaterials()[materialIndex]; + const Core::uint32 materialPresent = material != nullptr ? 1u : 0u; + output.write(reinterpret_cast(&materialPresent), sizeof(materialPresent)); + if (material != nullptr) { + WriteMaterialBlock(output, *material, textureFileNames); + } + } + + return static_cast(output); +} + +void DestroyImportedMesh(Mesh* mesh) { + if (mesh == nullptr) { + return; + } + + std::vector materials; + materials.reserve(mesh->GetMaterials().Size()); + for (Material* material : mesh->GetMaterials()) { + if (material != nullptr) { + materials.push_back(material); + } + } + + std::vector textures; + textures.reserve(mesh->GetTextures().Size()); + for (Texture* texture : mesh->GetTextures()) { + if (texture != nullptr) { + textures.push_back(texture); + } + } + + delete mesh; + for (Material* material : materials) { + delete material; + } + for (Texture* texture : textures) { + delete texture; + } +} + +} // namespace + +void AssetDatabase::Initialize(const Containers::String& projectRoot) { + m_projectRoot = NormalizePathString(projectRoot); + m_assetsRoot = NormalizePathString(fs::path(m_projectRoot.CStr()) / "Assets"); + m_libraryRoot = NormalizePathString(fs::path(m_projectRoot.CStr()) / "Library"); + m_sourceDbPath = NormalizePathString(fs::path(m_libraryRoot.CStr()) / "SourceAssetDB" / "assets.db"); + m_artifactDbPath = NormalizePathString(fs::path(m_libraryRoot.CStr()) / "ArtifactDB" / "artifacts.db"); + + EnsureProjectLayout(); + LoadSourceAssetDB(); + LoadArtifactDB(); + ScanAssets(); +} + +void AssetDatabase::Shutdown() { + SaveSourceAssetDB(); + SaveArtifactDB(); + + m_projectRoot.Clear(); + m_assetsRoot.Clear(); + m_libraryRoot.Clear(); + m_sourceDbPath.Clear(); + m_artifactDbPath.Clear(); + m_sourcesByPathKey.clear(); + m_sourcesByGuid.clear(); + m_artifactsByGuid.clear(); +} + +void AssetDatabase::Refresh() { + ScanAssets(); +} + +bool AssetDatabase::ResolvePath(const Containers::String& requestPath, + Containers::String& outAbsolutePath, + Containers::String& outRelativePath) const { + if (requestPath.Empty()) { + return false; + } + + fs::path inputPath(requestPath.CStr()); + if (inputPath.is_absolute()) { + outAbsolutePath = NormalizePathString(inputPath); + + std::error_code ec; + const fs::path projectRootPath(m_projectRoot.CStr()); + const fs::path relativePath = fs::relative(inputPath, projectRootPath, ec); + if (!ec) { + const Containers::String normalizedRelative = NormalizePathString(relativePath); + if (normalizedRelative.StartsWith("Assets/") || normalizedRelative == "Assets") { + outRelativePath = normalizedRelative; + } else { + outRelativePath.Clear(); + } + } else { + outRelativePath.Clear(); + } + return true; + } + + const Containers::String normalizedRequest = NormalizePathString(requestPath); + if (normalizedRequest.StartsWith("Assets/") || normalizedRequest == "Assets") { + outRelativePath = normalizedRequest; + outAbsolutePath = NormalizePathString(fs::path(m_projectRoot.CStr()) / normalizedRequest.CStr()); + return true; + } + + outAbsolutePath = NormalizePathString(fs::path(m_projectRoot.CStr()) / requestPath.CStr()); + outRelativePath.Clear(); + return true; +} + +bool AssetDatabase::TryGetAssetGuid(const Containers::String& requestPath, AssetGUID& outGuid) const { + Containers::String absolutePath; + Containers::String relativePath; + if (!ResolvePath(requestPath, absolutePath, relativePath) || relativePath.Empty()) { + return false; + } + + const auto sourceIt = m_sourcesByPathKey.find(ToStdString(MakeKey(relativePath))); + if (sourceIt == m_sourcesByPathKey.end()) { + return false; + } + + outGuid = sourceIt->second.guid; + return outGuid.IsValid(); +} + +bool AssetDatabase::TryGetAssetRef(const Containers::String& requestPath, + ResourceType resourceType, + AssetRef& outRef) const { + AssetGUID guid; + if (!TryGetAssetGuid(requestPath, guid)) { + return false; + } + + outRef.assetGuid = guid; + outRef.localID = kMainAssetLocalID; + outRef.resourceType = resourceType; + return true; +} + +bool AssetDatabase::TryGetPrimaryAssetPath(const AssetGUID& guid, Containers::String& outRelativePath) const { + const auto sourceIt = m_sourcesByGuid.find(guid); + if (sourceIt == m_sourcesByGuid.end()) { + return false; + } + + outRelativePath = sourceIt->second.relativePath; + return true; +} + +void AssetDatabase::EnsureProjectLayout() { + std::error_code ec; + fs::create_directories(fs::path(m_assetsRoot.CStr()), ec); + ec.clear(); + fs::create_directories(fs::path(m_libraryRoot.CStr()) / "SourceAssetDB", ec); + ec.clear(); + fs::create_directories(fs::path(m_libraryRoot.CStr()) / "ArtifactDB", ec); + ec.clear(); + fs::create_directories(fs::path(m_libraryRoot.CStr()) / "Artifacts", ec); +} + +void AssetDatabase::LoadSourceAssetDB() { + m_sourcesByPathKey.clear(); + m_sourcesByGuid.clear(); + + std::ifstream input(m_sourceDbPath.CStr()); + if (!input.is_open()) { + return; + } + + std::string line; + while (std::getline(input, line)) { + if (line.empty() || line[0] == '#') { + continue; + } + + const std::vector fields = SplitFields(line); + if (fields.size() < 10) { + continue; + } + + SourceAssetRecord record; + record.guid = AssetGUID::ParseOrDefault(ToContainersString(fields[0])); + record.relativePath = ToContainersString(fields[1]); + record.metaPath = ToContainersString(fields[2]); + record.isFolder = (fields[3] == "1"); + record.importerName = ToContainersString(fields[4]); + record.importerVersion = static_cast(std::stoul(fields[5])); + record.metaHash = ToContainersString(fields[6]); + record.sourceHash = ToContainersString(fields[7]); + record.sourceFileSize = static_cast(std::stoull(fields[8])); + record.sourceWriteTime = static_cast(std::stoull(fields[9])); + if (fields.size() > 10) { + record.lastKnownArtifactKey = ToContainersString(fields[10]); + } + + if (!record.guid.IsValid() || record.relativePath.Empty()) { + continue; + } + + m_sourcesByGuid[record.guid] = record; + m_sourcesByPathKey[ToStdString(MakeKey(record.relativePath))] = record; + } +} + +void AssetDatabase::SaveSourceAssetDB() const { + std::ofstream output(m_sourceDbPath.CStr(), std::ios::out | std::ios::trunc); + if (!output.is_open()) { + return; + } + + output << "# guid\trelativePath\tmetaPath\tisFolder\timporter\timporterVersion\tmetaHash\tsourceHash\tsize\twriteTime\tartifactKey\n"; + for (const auto& [guid, record] : m_sourcesByGuid) { + output << EscapeField(ToStdString(record.guid.ToString())) << '\t' + << EscapeField(ToStdString(record.relativePath)) << '\t' + << EscapeField(ToStdString(record.metaPath)) << '\t' + << (record.isFolder ? "1" : "0") << '\t' + << EscapeField(ToStdString(record.importerName)) << '\t' + << record.importerVersion << '\t' + << EscapeField(ToStdString(record.metaHash)) << '\t' + << EscapeField(ToStdString(record.sourceHash)) << '\t' + << record.sourceFileSize << '\t' + << record.sourceWriteTime << '\t' + << EscapeField(ToStdString(record.lastKnownArtifactKey)) << '\n'; + } +} + +void AssetDatabase::LoadArtifactDB() { + m_artifactsByGuid.clear(); + + std::ifstream input(m_artifactDbPath.CStr()); + if (!input.is_open()) { + return; + } + + std::string line; + while (std::getline(input, line)) { + if (line.empty() || line[0] == '#') { + continue; + } + + const std::vector fields = SplitFields(line); + if (fields.size() < 10) { + continue; + } + + ArtifactRecord record; + record.artifactKey = ToContainersString(fields[0]); + record.assetGuid = AssetGUID::ParseOrDefault(ToContainersString(fields[1])); + record.importerName = ToContainersString(fields[2]); + record.importerVersion = static_cast(std::stoul(fields[3])); + record.resourceType = static_cast(std::stoul(fields[4])); + record.artifactDirectory = ToContainersString(fields[5]); + record.mainArtifactPath = ToContainersString(fields[6]); + record.sourceHash = ToContainersString(fields[7]); + record.metaHash = ToContainersString(fields[8]); + record.sourceFileSize = static_cast(std::stoull(fields[9])); + record.sourceWriteTime = fields.size() > 10 ? static_cast(std::stoull(fields[10])) : 0; + record.mainLocalID = fields.size() > 11 ? static_cast(std::stoull(fields[11])) : kMainAssetLocalID; + + if (!record.assetGuid.IsValid() || record.artifactKey.Empty()) { + continue; + } + + m_artifactsByGuid[record.assetGuid] = record; + } +} + +void AssetDatabase::SaveArtifactDB() const { + std::ofstream output(m_artifactDbPath.CStr(), std::ios::out | std::ios::trunc); + if (!output.is_open()) { + return; + } + + output << "# artifactKey\tassetGuid\timporter\tversion\ttype\tartifactDir\tmainArtifact\tsourceHash\tmetaHash\tsize\twriteTime\tmainLocalID\n"; + for (const auto& [guid, record] : m_artifactsByGuid) { + output << EscapeField(ToStdString(record.artifactKey)) << '\t' + << EscapeField(ToStdString(record.assetGuid.ToString())) << '\t' + << EscapeField(ToStdString(record.importerName)) << '\t' + << record.importerVersion << '\t' + << static_cast(record.resourceType) << '\t' + << EscapeField(ToStdString(record.artifactDirectory)) << '\t' + << EscapeField(ToStdString(record.mainArtifactPath)) << '\t' + << EscapeField(ToStdString(record.sourceHash)) << '\t' + << EscapeField(ToStdString(record.metaHash)) << '\t' + << record.sourceFileSize << '\t' + << record.sourceWriteTime << '\t' + << record.mainLocalID << '\n'; + } +} + +void AssetDatabase::ScanAssets() { + std::unordered_map seenPaths; + const fs::path assetsRootPath(m_assetsRoot.CStr()); + if (fs::exists(assetsRootPath)) { + ScanAssetPath(assetsRootPath, seenPaths); + } + RemoveMissingRecords(seenPaths); + SaveSourceAssetDB(); +} + +void AssetDatabase::ScanAssetPath(const fs::path& path, + std::unordered_map& seenPaths) { + if (!fs::exists(path)) { + return; + } + + if (path.has_extension() && ToLowerCopy(path.extension().string()) == ".meta") { + return; + } + + const bool isFolder = fs::is_directory(path); + SourceAssetRecord record; + if (EnsureMetaForPath(path, isFolder, record)) { + seenPaths[ToStdString(MakeKey(record.relativePath))] = true; + } + + if (!isFolder) { + return; + } + + for (const auto& entry : fs::directory_iterator(path)) { + ScanAssetPath(entry.path(), seenPaths); + } +} + +void AssetDatabase::RemoveMissingRecords(const std::unordered_map& seenPaths) { + std::vector missingPathKeys; + for (const auto& [pathKey, record] : m_sourcesByPathKey) { + if (seenPaths.find(pathKey) == seenPaths.end()) { + missingPathKeys.push_back(pathKey); + } + } + + for (const std::string& pathKey : missingPathKeys) { + auto recordIt = m_sourcesByPathKey.find(pathKey); + if (recordIt == m_sourcesByPathKey.end()) { + continue; + } + + m_artifactsByGuid.erase(recordIt->second.guid); + m_sourcesByGuid.erase(recordIt->second.guid); + m_sourcesByPathKey.erase(recordIt); + } + + if (!missingPathKeys.empty()) { + SaveArtifactDB(); + } +} + +bool AssetDatabase::EnsureMetaForPath(const fs::path& sourcePath, + bool isFolder, + SourceAssetRecord& outRecord) { + const Containers::String relativePath = NormalizeRelativePath(sourcePath); + if (relativePath.Empty()) { + return false; + } + + const std::string pathKey = ToStdString(MakeKey(relativePath)); + auto existingIt = m_sourcesByPathKey.find(pathKey); + if (existingIt != m_sourcesByPathKey.end()) { + outRecord = existingIt->second; + } else { + outRecord = SourceAssetRecord(); + outRecord.relativePath = relativePath; + outRecord.importerName = GetImporterNameForPath(relativePath, isFolder); + outRecord.importerVersion = kCurrentImporterVersion; + } + + outRecord.relativePath = relativePath; + outRecord.isFolder = isFolder; + outRecord.importerName = GetImporterNameForPath(relativePath, isFolder); + outRecord.importerVersion = kCurrentImporterVersion; + + const fs::path metaPath(sourcePath.string() + ".meta"); + outRecord.metaPath = NormalizeRelativePath(metaPath); + + bool shouldRewriteMeta = false; + if (!fs::exists(metaPath) || !ReadMetaFile(metaPath, outRecord) || !outRecord.guid.IsValid()) { + if (!outRecord.guid.IsValid()) { + outRecord.guid = AssetGUID::Generate(); + } + shouldRewriteMeta = true; + } + + const auto duplicateGuidIt = m_sourcesByGuid.find(outRecord.guid); + if (duplicateGuidIt != m_sourcesByGuid.end() && + duplicateGuidIt->second.relativePath != relativePath) { + outRecord.guid = AssetGUID::Generate(); + shouldRewriteMeta = true; + } + + if (shouldRewriteMeta) { + WriteMetaFile(metaPath, outRecord); + } + + outRecord.metaHash = HashStringToAssetGUID(ReadWholeFileText(metaPath)).ToString(); + if (isFolder) { + outRecord.sourceHash.Clear(); + outRecord.sourceFileSize = 0; + outRecord.sourceWriteTime = 0; + } else { + const Core::uint64 fileSize = GetFileSizeValue(sourcePath); + const Core::uint64 writeTime = GetFileWriteTimeValue(sourcePath); + if (existingIt != m_sourcesByPathKey.end() && + existingIt->second.sourceFileSize == fileSize && + existingIt->second.sourceWriteTime == writeTime && + !existingIt->second.sourceHash.Empty()) { + outRecord.sourceHash = existingIt->second.sourceHash; + } else { + outRecord.sourceHash = ComputeFileHash(sourcePath); + } + outRecord.sourceFileSize = fileSize; + outRecord.sourceWriteTime = writeTime; + } + + m_sourcesByPathKey[pathKey] = outRecord; + m_sourcesByGuid[outRecord.guid] = outRecord; + return true; +} + +bool AssetDatabase::ReadMetaFile(const fs::path& metaPath, + SourceAssetRecord& inOutRecord) const { + std::ifstream input(metaPath); + if (!input.is_open()) { + return false; + } + + std::string line; + while (std::getline(input, line)) { + const size_t colonPos = line.find(':'); + if (colonPos == std::string::npos) { + continue; + } + + const std::string key = TrimCopy(line.substr(0, colonPos)); + const std::string value = TrimCopy(line.substr(colonPos + 1)); + if (key == "guid") { + inOutRecord.guid = AssetGUID::ParseOrDefault(ToContainersString(value)); + } else if (key == "folderAsset") { + inOutRecord.isFolder = ToLowerCopy(value) == "true"; + } else if (key == "importer") { + inOutRecord.importerName = ToContainersString(value); + } else if (key == "importerVersion") { + inOutRecord.importerVersion = static_cast(std::stoul(value)); + } + } + + if (inOutRecord.importerName.Empty()) { + inOutRecord.importerName = GetImporterNameForPath(inOutRecord.relativePath, inOutRecord.isFolder); + } + if (inOutRecord.importerVersion == 0) { + inOutRecord.importerVersion = kCurrentImporterVersion; + } + + return true; +} + +void AssetDatabase::WriteMetaFile(const fs::path& metaPath, + const SourceAssetRecord& record) const { + std::ofstream output(metaPath, std::ios::out | std::ios::trunc); + if (!output.is_open()) { + return; + } + + output << "fileFormatVersion: 1\n"; + output << "guid: " << record.guid.ToString().CStr() << "\n"; + output << "folderAsset: " << (record.isFolder ? "true" : "false") << "\n"; + output << "importer: " << record.importerName.CStr() << "\n"; + output << "importerVersion: " << record.importerVersion << "\n"; +} + +Containers::String AssetDatabase::NormalizeRelativePath(const fs::path& sourcePath) const { + std::error_code ec; + const fs::path projectRootPath(m_projectRoot.CStr()); + const fs::path relativePath = fs::relative(sourcePath, projectRootPath, ec); + if (ec) { + return Containers::String(); + } + + return NormalizePathString(relativePath); +} + +Containers::String AssetDatabase::NormalizePathString(const fs::path& path) { + return ToContainersString(path.lexically_normal().generic_string()); +} + +Containers::String AssetDatabase::NormalizePathString(const Containers::String& path) { + return NormalizePathString(fs::path(path.CStr())); +} + +Containers::String AssetDatabase::MakeKey(const Containers::String& path) { + std::string key = ToStdString(NormalizePathString(path)); + std::transform(key.begin(), key.end(), key.begin(), [](unsigned char ch) { + return static_cast(std::tolower(ch)); + }); + return ToContainersString(key); +} + +Containers::String AssetDatabase::GetImporterNameForPath(const Containers::String& relativePath, bool isFolder) { + if (isFolder) { + return Containers::String("FolderImporter"); + } + + const std::string ext = ToLowerCopy(fs::path(relativePath.CStr()).extension().string()); + if (ext == ".png" || ext == ".jpg" || ext == ".jpeg" || ext == ".bmp" || ext == ".tga" || ext == ".gif" || ext == ".hdr") { + return Containers::String("TextureImporter"); + } + if (ext == ".obj" || ext == ".fbx" || ext == ".gltf" || ext == ".glb" || ext == ".dae" || ext == ".stl") { + return Containers::String("ModelImporter"); + } + if (ext == ".mat" || ext == ".material" || ext == ".json") { + return Containers::String("MaterialImporter"); + } + return Containers::String("DefaultImporter"); +} + +ResourceType AssetDatabase::GetPrimaryResourceTypeForImporter(const Containers::String& importerName) { + if (importerName == "TextureImporter") { + return ResourceType::Texture; + } + if (importerName == "ModelImporter") { + return ResourceType::Mesh; + } + if (importerName == "MaterialImporter") { + return ResourceType::Material; + } + return ResourceType::Unknown; +} + +bool AssetDatabase::ShouldReimport(const SourceAssetRecord& sourceRecord, + const ArtifactRecord* artifactRecord) const { + if (artifactRecord == nullptr) { + return true; + } + + if (artifactRecord->artifactKey.Empty() || + artifactRecord->mainArtifactPath.Empty()) { + return true; + } + + const fs::path artifactMainPath = fs::path(m_projectRoot.CStr()) / artifactRecord->mainArtifactPath.CStr(); + if (!fs::exists(artifactMainPath)) { + return true; + } + + return artifactRecord->sourceHash != sourceRecord.sourceHash || + artifactRecord->metaHash != sourceRecord.metaHash || + artifactRecord->sourceFileSize != sourceRecord.sourceFileSize || + artifactRecord->sourceWriteTime != sourceRecord.sourceWriteTime; +} + +bool AssetDatabase::ImportAsset(const SourceAssetRecord& sourceRecord, + ArtifactRecord& outRecord) { + const ResourceType primaryType = GetPrimaryResourceTypeForImporter(sourceRecord.importerName); + switch (primaryType) { + case ResourceType::Texture: + return ImportTextureAsset(sourceRecord, outRecord); + case ResourceType::Mesh: + return ImportModelAsset(sourceRecord, outRecord); + default: + return false; + } +} + +bool AssetDatabase::EnsureArtifact(const Containers::String& requestPath, + ResourceType requestedType, + ResolvedAsset& outAsset) { + outAsset = ResolvedAsset(); + + Containers::String absolutePath; + Containers::String relativePath; + if (!ResolvePath(requestPath, absolutePath, relativePath) || relativePath.Empty()) { + return false; + } + + const fs::path absoluteFsPath(absolutePath.CStr()); + if (!fs::exists(absoluteFsPath)) { + return false; + } + + SourceAssetRecord sourceRecord; + if (!EnsureMetaForPath(absoluteFsPath, fs::is_directory(absoluteFsPath), sourceRecord)) { + return false; + } + + const ResourceType primaryType = GetPrimaryResourceTypeForImporter(sourceRecord.importerName); + if (primaryType == ResourceType::Unknown || primaryType != requestedType) { + return false; + } + + ArtifactRecord* artifactRecord = nullptr; + auto artifactIt = m_artifactsByGuid.find(sourceRecord.guid); + if (artifactIt != m_artifactsByGuid.end()) { + artifactRecord = &artifactIt->second; + } + + if (ShouldReimport(sourceRecord, artifactRecord)) { + ArtifactRecord rebuiltRecord; + if (!ImportAsset(sourceRecord, rebuiltRecord)) { + return false; + } + + m_artifactsByGuid[sourceRecord.guid] = rebuiltRecord; + m_sourcesByGuid[sourceRecord.guid].lastKnownArtifactKey = rebuiltRecord.artifactKey; + m_sourcesByPathKey[ToStdString(MakeKey(sourceRecord.relativePath))].lastKnownArtifactKey = rebuiltRecord.artifactKey; + SaveArtifactDB(); + SaveSourceAssetDB(); + artifactRecord = &m_artifactsByGuid[sourceRecord.guid]; + } + + if (artifactRecord == nullptr) { + return false; + } + + outAsset.exists = true; + outAsset.artifactReady = true; + outAsset.absolutePath = absolutePath; + outAsset.relativePath = sourceRecord.relativePath; + outAsset.assetGuid = sourceRecord.guid; + outAsset.resourceType = artifactRecord->resourceType; + outAsset.artifactDirectory = NormalizePathString(fs::path(m_projectRoot.CStr()) / artifactRecord->artifactDirectory.CStr()); + outAsset.artifactMainPath = NormalizePathString(fs::path(m_projectRoot.CStr()) / artifactRecord->mainArtifactPath.CStr()); + outAsset.mainLocalID = artifactRecord->mainLocalID; + return true; +} + +bool AssetDatabase::ImportTextureAsset(const SourceAssetRecord& sourceRecord, + ArtifactRecord& outRecord) { + TextureLoader loader; + const Containers::String absolutePath = NormalizePathString(fs::path(m_projectRoot.CStr()) / sourceRecord.relativePath.CStr()); + LoadResult result = loader.Load(absolutePath); + if (!result || result.resource == nullptr) { + return false; + } + + Texture* texture = static_cast(result.resource); + const Containers::String artifactKey = BuildArtifactKey(sourceRecord); + const Containers::String artifactDir = BuildArtifactDirectory(artifactKey); + const Containers::String mainArtifactPath = NormalizePathString(fs::path(artifactDir.CStr()) / "main.xctex"); + + std::error_code ec; + fs::remove_all(fs::path(m_projectRoot.CStr()) / artifactDir.CStr(), ec); + ec.clear(); + fs::create_directories(fs::path(m_projectRoot.CStr()) / artifactDir.CStr(), ec); + if (ec) { + delete texture; + return false; + } + + const bool writeOk = WriteTextureArtifactFile(fs::path(m_projectRoot.CStr()) / mainArtifactPath.CStr(), *texture); + delete texture; + if (!writeOk) { + return false; + } + + outRecord.artifactKey = artifactKey; + outRecord.assetGuid = sourceRecord.guid; + outRecord.importerName = sourceRecord.importerName; + outRecord.importerVersion = sourceRecord.importerVersion; + outRecord.resourceType = ResourceType::Texture; + outRecord.artifactDirectory = artifactDir; + outRecord.mainArtifactPath = mainArtifactPath; + outRecord.sourceHash = sourceRecord.sourceHash; + outRecord.metaHash = sourceRecord.metaHash; + outRecord.sourceFileSize = sourceRecord.sourceFileSize; + outRecord.sourceWriteTime = sourceRecord.sourceWriteTime; + outRecord.mainLocalID = kMainAssetLocalID; + return true; +} + +bool AssetDatabase::ImportModelAsset(const SourceAssetRecord& sourceRecord, + ArtifactRecord& outRecord) { + MeshLoader loader; + const Containers::String absolutePath = NormalizePathString(fs::path(m_projectRoot.CStr()) / sourceRecord.relativePath.CStr()); + LoadResult result = loader.Load(absolutePath); + if (!result || result.resource == nullptr) { + return false; + } + + Mesh* mesh = static_cast(result.resource); + const Containers::String artifactKey = BuildArtifactKey(sourceRecord); + const Containers::String artifactDir = BuildArtifactDirectory(artifactKey); + const Containers::String mainArtifactPath = NormalizePathString(fs::path(artifactDir.CStr()) / "main.xcmesh"); + + std::error_code ec; + fs::remove_all(fs::path(m_projectRoot.CStr()) / artifactDir.CStr(), ec); + ec.clear(); + fs::create_directories(fs::path(m_projectRoot.CStr()) / artifactDir.CStr(), ec); + if (ec) { + DestroyImportedMesh(mesh); + return false; + } + + bool writeOk = WriteMeshArtifactFile(fs::path(m_projectRoot.CStr()) / mainArtifactPath.CStr(), *mesh); + for (size_t textureIndex = 0; writeOk && textureIndex < mesh->GetTextures().Size(); ++textureIndex) { + Texture* texture = mesh->GetTextures()[textureIndex]; + if (texture == nullptr) { + continue; + } + + const fs::path textureArtifactPath = + fs::path(m_projectRoot.CStr()) / artifactDir.CStr() / ("texture_" + std::to_string(textureIndex) + ".xctex"); + writeOk = WriteTextureArtifactFile(textureArtifactPath, *texture); + } + + DestroyImportedMesh(mesh); + if (!writeOk) { + return false; + } + + outRecord.artifactKey = artifactKey; + outRecord.assetGuid = sourceRecord.guid; + outRecord.importerName = sourceRecord.importerName; + outRecord.importerVersion = sourceRecord.importerVersion; + outRecord.resourceType = ResourceType::Mesh; + outRecord.artifactDirectory = artifactDir; + outRecord.mainArtifactPath = mainArtifactPath; + outRecord.sourceHash = sourceRecord.sourceHash; + outRecord.metaHash = sourceRecord.metaHash; + outRecord.sourceFileSize = sourceRecord.sourceFileSize; + outRecord.sourceWriteTime = sourceRecord.sourceWriteTime; + outRecord.mainLocalID = kMainAssetLocalID; + return true; +} + +Containers::String AssetDatabase::BuildArtifactKey(const SourceAssetRecord& sourceRecord) const { + Containers::String signature = sourceRecord.guid.ToString(); + signature += ":"; + signature += sourceRecord.importerName; + signature += ":"; + signature += Containers::String(std::to_string(sourceRecord.importerVersion).c_str()); + signature += ":"; + signature += sourceRecord.sourceHash; + signature += ":"; + signature += sourceRecord.metaHash; + signature += ":"; + signature += Containers::String(std::to_string(sourceRecord.sourceFileSize).c_str()); + signature += ":"; + signature += Containers::String(std::to_string(sourceRecord.sourceWriteTime).c_str()); + return HashStringToAssetGUID(signature).ToString(); +} + +Containers::String AssetDatabase::BuildArtifactDirectory(const Containers::String& artifactKey) const { + if (artifactKey.Length() < 2) { + return Containers::String("Library/Artifacts/00/invalid"); + } + + const Containers::String shard = artifactKey.Substring(0, 2); + return Containers::String("Library/Artifacts/") + shard + "/" + artifactKey; +} + +Containers::String AssetDatabase::ReadWholeFileText(const fs::path& path) { + std::ifstream input(path, std::ios::binary); + if (!input.is_open()) { + return Containers::String(); + } + + std::ostringstream buffer; + buffer << input.rdbuf(); + return ToContainersString(buffer.str()); +} + +Containers::String AssetDatabase::ComputeFileHash(const fs::path& path) { + std::ifstream input(path, std::ios::binary); + if (!input.is_open()) { + return Containers::String(); + } + + std::vector bytes( + (std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + if (bytes.empty()) { + return HashBytesToAssetGUID(nullptr, 0).ToString(); + } + + return HashBytesToAssetGUID(bytes.data(), bytes.size()).ToString(); +} + +Core::uint64 AssetDatabase::GetFileSizeValue(const fs::path& path) { + std::error_code ec; + const auto size = fs::file_size(path, ec); + return ec ? 0 : static_cast(size); +} + +Core::uint64 AssetDatabase::GetFileWriteTimeValue(const fs::path& path) { + std::error_code ec; + const auto writeTime = fs::last_write_time(path, ec); + if (ec) { + return 0; + } + + return static_cast(writeTime.time_since_epoch().count()); +} + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/src/Core/Asset/AssetGUID.cpp b/engine/src/Core/Asset/AssetGUID.cpp new file mode 100644 index 00000000..e4d3f169 --- /dev/null +++ b/engine/src/Core/Asset/AssetGUID.cpp @@ -0,0 +1,114 @@ +#include + +#include +#include + +namespace XCEngine { +namespace Resources { + +namespace { + +Core::uint64 FNV1a64(const Core::uint8* bytes, size_t size, Core::uint64 seed) { + Core::uint64 hash = seed; + for (size_t index = 0; index < size; ++index) { + hash ^= static_cast(bytes[index]); + hash *= 1099511628211ULL; + } + return hash; +} + +int HexDigitToInt(char ch) { + if (ch >= '0' && ch <= '9') { + return ch - '0'; + } + if (ch >= 'a' && ch <= 'f') { + return 10 + (ch - 'a'); + } + if (ch >= 'A' && ch <= 'F') { + return 10 + (ch - 'A'); + } + return -1; +} + +bool ParseHex64(const char* text, Core::uint64& outValue) { + outValue = 0; + for (int index = 0; index < 16; ++index) { + const int digit = HexDigitToInt(text[index]); + if (digit < 0) { + return false; + } + outValue = (outValue << 4) | static_cast(digit); + } + return true; +} + +} // namespace + +AssetGUID AssetGUID::Generate() { + static std::random_device rd; + static std::mt19937_64 generator(rd()); + + AssetGUID guid; + do { + guid.high = generator(); + guid.low = generator(); + } while (!guid.IsValid()); + + return guid; +} + +bool AssetGUID::TryParse(const Containers::String& text, AssetGUID& outGuid) { + if (text.Length() != 32) { + return false; + } + + Core::uint64 highValue = 0; + Core::uint64 lowValue = 0; + if (!ParseHex64(text.CStr(), highValue) || + !ParseHex64(text.CStr() + 16, lowValue)) { + return false; + } + + outGuid = AssetGUID(highValue, lowValue); + return true; +} + +AssetGUID AssetGUID::ParseOrDefault(const Containers::String& text) { + AssetGUID guid; + if (TryParse(text, guid)) { + return guid; + } + return AssetGUID(); +} + +Containers::String AssetGUID::ToString() const { + char buffer[33] = {}; +#if defined(_MSC_VER) + std::snprintf(buffer, sizeof(buffer), "%016llx%016llx", + static_cast(high), + static_cast(low)); +#else + std::snprintf(buffer, sizeof(buffer), "%016lx%016lx", + static_cast(high), + static_cast(low)); +#endif + return Containers::String(buffer); +} + +AssetGUID HashBytesToAssetGUID(const void* data, size_t size) { + if (data == nullptr || size == 0) { + return AssetGUID(); + } + + const auto* bytes = static_cast(data); + const Core::uint64 highHash = FNV1a64(bytes, size, 14695981039346656037ULL); + const Core::uint64 lowHash = FNV1a64(bytes, size, 1099511628211ULL ^ 0x9e3779b97f4a7c15ULL); + return AssetGUID(highHash, lowHash); +} + +AssetGUID HashStringToAssetGUID(const Containers::String& text) { + return HashBytesToAssetGUID(text.CStr(), text.Length()); +} + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/src/Core/Asset/ResourceManager.cpp b/engine/src/Core/Asset/ResourceManager.cpp index a240da41..be80ce8b 100644 --- a/engine/src/Core/Asset/ResourceManager.cpp +++ b/engine/src/Core/Asset/ResourceManager.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,10 @@ ResourceManager& ResourceManager::Get() { } void ResourceManager::Initialize() { + if (m_asyncLoader) { + return; + } + m_asyncLoader = Core::MakeUnique(); m_asyncLoader->Initialize(2); @@ -42,12 +47,23 @@ void ResourceManager::Initialize() { void ResourceManager::Shutdown() { UnloadAll(); - m_asyncLoader->Shutdown(); - m_asyncLoader.reset(); + if (m_asyncLoader) { + m_asyncLoader->Shutdown(); + m_asyncLoader.reset(); + } + m_assetDatabase.Shutdown(); + ResourceFileSystem::Get().Shutdown(); } void ResourceManager::SetResourceRoot(const Containers::String& rootPath) { m_resourceRoot = rootPath; + if (!m_resourceRoot.Empty()) { + ResourceFileSystem::Get().Initialize(rootPath); + m_assetDatabase.Initialize(rootPath); + } else { + ResourceFileSystem::Get().Shutdown(); + m_assetDatabase.Shutdown(); + } } const Containers::String& ResourceManager::GetResourceRoot() const { @@ -111,7 +127,8 @@ IResource* ResourceManager::FindInCache(ResourceGUID guid) { void ResourceManager::AddToCache(ResourceGUID guid, IResource* resource) { std::lock_guard lock(m_mutex); - + + resource->m_guid = guid; m_resourceCache.Insert(guid, resource); m_memoryUsage += resource->GetMemorySize(); m_cache.Add(guid, resource); @@ -128,6 +145,7 @@ void ResourceManager::Unload(ResourceGUID guid) { if (it != nullptr) { IResource* resource = *it; m_resourceCache.Erase(guid); + m_guidToPath.Erase(guid); m_memoryUsage -= resource->GetMemorySize(); resource->Release(); } @@ -135,13 +153,16 @@ void ResourceManager::Unload(ResourceGUID guid) { void ResourceManager::UnloadAll() { std::lock_guard lock(m_mutex); - - for (size_t i = 0; i < m_resourceCache.Size(); ++i) { - // This is a simplified approach - we'd need a way to iterate - // For now, just clear everything + + const auto cachedResources = m_resourceCache.GetPairs(); + for (const auto& pair : cachedResources) { + if (pair.second != nullptr) { + pair.second->Release(); + } } m_resourceCache.Clear(); m_refCounts.Clear(); + m_guidToPath.Clear(); m_memoryUsage = 0; } @@ -209,6 +230,10 @@ void ResourceManager::ReloadResource(ResourceGUID guid) { if (pathIt == nullptr) { return; } + + const Containers::String path = *pathIt; + auto* typeIt = m_resourceCache.Find(guid); + (void)typeIt; } Containers::Array ResourceManager::GetResourcePaths() const { @@ -226,11 +251,60 @@ void ResourceManager::UnloadGroup(const Containers::Array& guids) if (it != nullptr) { IResource* resource = *it; m_resourceCache.Erase(guid); + m_guidToPath.Erase(guid); m_memoryUsage -= resource->GetMemorySize(); resource->Release(); } } } +void ResourceManager::RefreshAssetDatabase() { + if (!m_resourceRoot.Empty()) { + m_assetDatabase.Refresh(); + } +} + +bool ResourceManager::TryGetAssetRef(const Containers::String& path, ResourceType resourceType, AssetRef& outRef) const { + return m_assetDatabase.TryGetAssetRef(path, resourceType, outRef); +} + +LoadResult ResourceManager::LoadResource(const Containers::String& path, + ResourceType type, + ImportSettings* settings) { + const ResourceGUID guid = ResourceGUID::Generate(path); + + if (IResource* cached = FindInCache(guid)) { + return LoadResult(cached); + } + + IResourceLoader* loader = FindLoader(type); + if (loader == nullptr) { + Debug::Logger::Get().Warning(Debug::LogCategory::FileSystem, + Containers::String("No loader found for resource type: ") + + GetResourceTypeName(type)); + return LoadResult(false, "Loader not found"); + } + + Containers::String loadPath = path; + AssetDatabase::ResolvedAsset resolvedAsset; + if (!m_resourceRoot.Empty() && + m_assetDatabase.EnsureArtifact(path, type, resolvedAsset) && + resolvedAsset.artifactReady) { + loadPath = resolvedAsset.artifactMainPath; + } + + LoadResult result = loader->Load(loadPath, settings); + if (!result || result.resource == nullptr) { + Debug::Logger::Get().Error(Debug::LogCategory::FileSystem, + Containers::String("Failed to load resource: ") + path + " - " + result.errorMessage); + return result; + } + + result.resource->m_path = path; + AddToCache(guid, result.resource); + m_guidToPath.Insert(guid, path); + return result; +} + } // namespace Resources } // namespace XCEngine diff --git a/engine/src/Core/IO/IResourceLoader.cpp b/engine/src/Core/IO/IResourceLoader.cpp index a2bb4a3f..00129948 100644 --- a/engine/src/Core/IO/IResourceLoader.cpp +++ b/engine/src/Core/IO/IResourceLoader.cpp @@ -1,28 +1,57 @@ #include +#include +#include #include namespace XCEngine { namespace Resources { -Containers::Array IResourceLoader::ReadFileData(const Containers::String& path) { +namespace { + +Containers::Array TryReadFileData( + const std::filesystem::path& filePath, + bool& opened) { Containers::Array data; - - std::ifstream file(path.CStr(), std::ios::binary | std::ios::ate); + + std::ifstream file(filePath, std::ios::binary | std::ios::ate); if (!file.is_open()) { + opened = false; return data; } - - std::streamsize size = file.tellg(); + + opened = true; + const std::streamsize size = file.tellg(); + if (size <= 0) { + return data; + } + file.seekg(0, std::ios::beg); - data.Resize(static_cast(size)); if (!file.read(reinterpret_cast(data.Data()), size)) { data.Clear(); } - + return data; } +} // namespace + +Containers::Array IResourceLoader::ReadFileData(const Containers::String& path) { + bool opened = false; + const std::filesystem::path inputPath(path.CStr()); + Containers::Array data = TryReadFileData(inputPath, opened); + if (opened || path.Empty() || inputPath.is_absolute()) { + return data; + } + + const Containers::String& resourceRoot = ResourceManager::Get().GetResourceRoot(); + if (resourceRoot.Empty()) { + return data; + } + + return TryReadFileData(std::filesystem::path(resourceRoot.CStr()) / inputPath, opened); +} + Containers::String IResourceLoader::GetExtension(const Containers::String& path) { Containers::String ext; size_t dotPos = Containers::String::npos; diff --git a/engine/src/Resources/BuiltinResources.cpp b/engine/src/Resources/BuiltinResources.cpp new file mode 100644 index 00000000..0ef7acd1 --- /dev/null +++ b/engine/src/Resources/BuiltinResources.cpp @@ -0,0 +1,677 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace XCEngine { +namespace Resources { + +namespace { + +constexpr const char* kBuiltinPrefix = "builtin://"; +constexpr const char* kBuiltinMeshPrefix = "builtin://meshes/"; +constexpr const char* kBuiltinMaterialPrefix = "builtin://materials/"; +constexpr const char* kBuiltinTexturePrefix = "builtin://textures/"; +constexpr const char* kBuiltinDefaultPrimitiveMaterialPath = "builtin://materials/default-primitive"; +constexpr const char* kBuiltinDefaultPrimitiveTexturePath = "builtin://textures/default-primitive-albedo"; +constexpr float kPi = 3.14159265358979323846f; + +struct MeshBuffers { + std::vector vertices; + std::vector indices; +}; + +Math::Bounds ComputeBounds(const std::vector& vertices) { + if (vertices.empty()) { + return Math::Bounds(); + } + + Math::Vector3 min = vertices.front().position; + Math::Vector3 max = vertices.front().position; + for (const StaticMeshVertex& vertex : vertices) { + min.x = std::min(min.x, vertex.position.x); + min.y = std::min(min.y, vertex.position.y); + min.z = std::min(min.z, vertex.position.z); + max.x = std::max(max.x, vertex.position.x); + max.y = std::max(max.y, vertex.position.y); + max.z = std::max(max.z, vertex.position.z); + } + + Math::Bounds bounds; + bounds.SetMinMax(min, max); + return bounds; +} + +StaticMeshVertex MakeVertex( + const Math::Vector3& position, + const Math::Vector3& normal, + const Math::Vector3& tangent, + const Math::Vector2& uv) { + StaticMeshVertex vertex; + vertex.position = position; + vertex.normal = normal.Normalized(); + vertex.tangent = tangent.Normalized(); + vertex.bitangent = Math::Vector3::Cross(vertex.normal, vertex.tangent).Normalized(); + vertex.uv0 = uv; + return vertex; +} + +void AppendQuad( + MeshBuffers& buffers, + const Math::Vector3& bottomLeft, + const Math::Vector3& bottomRight, + const Math::Vector3& topRight, + const Math::Vector3& topLeft, + const Math::Vector3& normal, + const Math::Vector3& tangent) { + const Core::uint32 baseIndex = static_cast(buffers.vertices.size()); + buffers.vertices.push_back(MakeVertex(bottomLeft, normal, tangent, Math::Vector2(0.0f, 0.0f))); + buffers.vertices.push_back(MakeVertex(bottomRight, normal, tangent, Math::Vector2(1.0f, 0.0f))); + buffers.vertices.push_back(MakeVertex(topRight, normal, tangent, Math::Vector2(1.0f, 1.0f))); + buffers.vertices.push_back(MakeVertex(topLeft, normal, tangent, Math::Vector2(0.0f, 1.0f))); + + buffers.indices.push_back(baseIndex + 0); + buffers.indices.push_back(baseIndex + 1); + buffers.indices.push_back(baseIndex + 2); + buffers.indices.push_back(baseIndex + 0); + buffers.indices.push_back(baseIndex + 2); + buffers.indices.push_back(baseIndex + 3); +} + +void FlipTriangleWinding(MeshBuffers& buffers) { + for (size_t index = 0; index + 2 < buffers.indices.size(); index += 3) { + std::swap(buffers.indices[index + 1], buffers.indices[index + 2]); + } +} + +MeshBuffers CreateCubeMeshBuffers() { + MeshBuffers buffers; + const float half = 0.5f; + + AppendQuad( + buffers, + Math::Vector3(-half, -half, half), + Math::Vector3(half, -half, half), + Math::Vector3(half, half, half), + Math::Vector3(-half, half, half), + Math::Vector3::Forward(), + Math::Vector3::Right()); + AppendQuad( + buffers, + Math::Vector3(half, -half, -half), + Math::Vector3(-half, -half, -half), + Math::Vector3(-half, half, -half), + Math::Vector3(half, half, -half), + Math::Vector3::Back(), + Math::Vector3::Left()); + AppendQuad( + buffers, + Math::Vector3(-half, -half, -half), + Math::Vector3(-half, -half, half), + Math::Vector3(-half, half, half), + Math::Vector3(-half, half, -half), + Math::Vector3::Left(), + Math::Vector3::Forward()); + AppendQuad( + buffers, + Math::Vector3(half, -half, half), + Math::Vector3(half, -half, -half), + Math::Vector3(half, half, -half), + Math::Vector3(half, half, half), + Math::Vector3::Right(), + Math::Vector3::Back()); + AppendQuad( + buffers, + Math::Vector3(-half, half, half), + Math::Vector3(half, half, half), + Math::Vector3(half, half, -half), + Math::Vector3(-half, half, -half), + Math::Vector3::Up(), + Math::Vector3::Right()); + AppendQuad( + buffers, + Math::Vector3(-half, -half, -half), + Math::Vector3(half, -half, -half), + Math::Vector3(half, -half, half), + Math::Vector3(-half, -half, half), + Math::Vector3::Down(), + Math::Vector3::Right()); + + return buffers; +} + +MeshBuffers CreateQuadMeshBuffers() { + MeshBuffers buffers; + AppendQuad( + buffers, + Math::Vector3(-0.5f, -0.5f, 0.0f), + Math::Vector3(0.5f, -0.5f, 0.0f), + Math::Vector3(0.5f, 0.5f, 0.0f), + Math::Vector3(-0.5f, 0.5f, 0.0f), + Math::Vector3::Forward(), + Math::Vector3::Right()); + return buffers; +} + +MeshBuffers CreatePlaneMeshBuffers() { + MeshBuffers buffers; + constexpr int kSegments = 10; + constexpr float kSize = 10.0f; + const float halfSize = kSize * 0.5f; + const float step = kSize / static_cast(kSegments); + + for (int z = 0; z <= kSegments; ++z) { + for (int x = 0; x <= kSegments; ++x) { + const float px = -halfSize + static_cast(x) * step; + const float pz = -halfSize + static_cast(z) * step; + const float u = static_cast(x) / static_cast(kSegments); + const float v = static_cast(z) / static_cast(kSegments); + buffers.vertices.push_back(MakeVertex( + Math::Vector3(px, 0.0f, pz), + Math::Vector3::Up(), + Math::Vector3::Right(), + Math::Vector2(u, v))); + } + } + + const int rowStride = kSegments + 1; + for (int z = 0; z < kSegments; ++z) { + for (int x = 0; x < kSegments; ++x) { + const Core::uint32 i0 = static_cast(z * rowStride + x); + const Core::uint32 i1 = i0 + 1; + const Core::uint32 i2 = i0 + static_cast(rowStride); + const Core::uint32 i3 = i2 + 1; + + buffers.indices.push_back(i0); + buffers.indices.push_back(i3); + buffers.indices.push_back(i1); + buffers.indices.push_back(i0); + buffers.indices.push_back(i2); + buffers.indices.push_back(i3); + } + } + + return buffers; +} + +MeshBuffers CreateUvSphereMeshBuffers() { + MeshBuffers buffers; + constexpr int kLongitudeSegments = 24; + constexpr int kLatitudeSegments = 16; + constexpr float kRadius = 0.5f; + + for (int latitude = 0; latitude <= kLatitudeSegments; ++latitude) { + const float v = static_cast(latitude) / static_cast(kLatitudeSegments); + const float theta = v * kPi; + const float sinTheta = std::sin(theta); + const float cosTheta = std::cos(theta); + + for (int longitude = 0; longitude <= kLongitudeSegments; ++longitude) { + const float u = static_cast(longitude) / static_cast(kLongitudeSegments); + const float phi = u * (2.0f * kPi); + const float sinPhi = std::sin(phi); + const float cosPhi = std::cos(phi); + + const Math::Vector3 normal(cosPhi * sinTheta, cosTheta, sinPhi * sinTheta); + Math::Vector3 tangent(-sinPhi, 0.0f, cosPhi); + if (tangent.SqrMagnitude() <= 0.000001f) { + tangent = Math::Vector3::Right(); + } + + buffers.vertices.push_back(MakeVertex( + normal * kRadius, + normal, + tangent, + Math::Vector2(u, 1.0f - v))); + } + } + + const int stride = kLongitudeSegments + 1; + for (int latitude = 0; latitude < kLatitudeSegments; ++latitude) { + for (int longitude = 0; longitude < kLongitudeSegments; ++longitude) { + const Core::uint32 i0 = static_cast(latitude * stride + longitude); + const Core::uint32 i1 = i0 + 1; + const Core::uint32 i2 = i0 + static_cast(stride); + const Core::uint32 i3 = i2 + 1; + + buffers.indices.push_back(i0); + buffers.indices.push_back(i2); + buffers.indices.push_back(i1); + buffers.indices.push_back(i1); + buffers.indices.push_back(i2); + buffers.indices.push_back(i3); + } + } + + return buffers; +} + +MeshBuffers CreateCylinderMeshBuffers() { + MeshBuffers buffers; + constexpr int kRadialSegments = 24; + constexpr float kRadius = 0.5f; + constexpr float kHalfHeight = 1.0f; + + for (int ring = 0; ring <= 1; ++ring) { + const float y = ring == 0 ? -kHalfHeight : kHalfHeight; + const float v = static_cast(ring); + for (int segment = 0; segment <= kRadialSegments; ++segment) { + const float u = static_cast(segment) / static_cast(kRadialSegments); + const float angle = u * (2.0f * kPi); + const float cosAngle = std::cos(angle); + const float sinAngle = std::sin(angle); + const Math::Vector3 normal(cosAngle, 0.0f, sinAngle); + const Math::Vector3 tangent(-sinAngle, 0.0f, cosAngle); + buffers.vertices.push_back(MakeVertex( + Math::Vector3(cosAngle * kRadius, y, sinAngle * kRadius), + normal, + tangent, + Math::Vector2(u, v))); + } + } + + const int sideStride = kRadialSegments + 1; + for (int segment = 0; segment < kRadialSegments; ++segment) { + const Core::uint32 i0 = static_cast(segment); + const Core::uint32 i1 = i0 + 1; + const Core::uint32 i2 = i0 + static_cast(sideStride); + const Core::uint32 i3 = i2 + 1; + + buffers.indices.push_back(i0); + buffers.indices.push_back(i2); + buffers.indices.push_back(i1); + buffers.indices.push_back(i1); + buffers.indices.push_back(i2); + buffers.indices.push_back(i3); + } + + const auto appendCap = [&](bool topCap) { + const float y = topCap ? kHalfHeight : -kHalfHeight; + const Math::Vector3 normal = topCap ? Math::Vector3::Up() : Math::Vector3::Down(); + const Core::uint32 centerIndex = static_cast(buffers.vertices.size()); + buffers.vertices.push_back(MakeVertex( + Math::Vector3(0.0f, y, 0.0f), + normal, + Math::Vector3::Right(), + Math::Vector2(0.5f, 0.5f))); + + for (int segment = 0; segment <= kRadialSegments; ++segment) { + const float u = static_cast(segment) / static_cast(kRadialSegments); + const float angle = u * (2.0f * kPi); + const float cosAngle = std::cos(angle); + const float sinAngle = std::sin(angle); + buffers.vertices.push_back(MakeVertex( + Math::Vector3(cosAngle * kRadius, y, sinAngle * kRadius), + normal, + Math::Vector3::Right(), + Math::Vector2(cosAngle * 0.5f + 0.5f, sinAngle * 0.5f + 0.5f))); + } + + for (int segment = 0; segment < kRadialSegments; ++segment) { + const Core::uint32 rim0 = centerIndex + 1 + static_cast(segment); + const Core::uint32 rim1 = rim0 + 1; + if (topCap) { + buffers.indices.push_back(centerIndex); + buffers.indices.push_back(rim0); + buffers.indices.push_back(rim1); + } else { + buffers.indices.push_back(centerIndex); + buffers.indices.push_back(rim1); + buffers.indices.push_back(rim0); + } + } + }; + + appendCap(true); + appendCap(false); + return buffers; +} + +MeshBuffers CreateCapsuleMeshBuffers() { + MeshBuffers buffers; + constexpr int kRadialSegments = 24; + constexpr int kHemisphereSegments = 8; + constexpr float kRadius = 0.5f; + constexpr float kHalfCylinderHeight = 0.5f; + + struct RingDefinition { + float y = 0.0f; + float radius = 0.0f; + Math::Vector3 normalBase = Math::Vector3::Zero(); + }; + + std::vector rings; + rings.reserve(static_cast(kHemisphereSegments * 2 + 2)); + + for (int step = 0; step <= kHemisphereSegments; ++step) { + const float t = static_cast(step) / static_cast(kHemisphereSegments); + const float angle = -0.5f * kPi + t * (0.5f * kPi); + const float ringRadius = std::cos(angle) * kRadius; + const float y = std::sin(angle) * kRadius - kHalfCylinderHeight; + const Math::Vector3 normalBase(0.0f, std::sin(angle), 0.0f); + rings.push_back({ y, ringRadius, normalBase }); + } + + for (int step = 0; step <= kHemisphereSegments; ++step) { + const float t = static_cast(step) / static_cast(kHemisphereSegments); + const float angle = t * (0.5f * kPi); + const float ringRadius = std::cos(angle) * kRadius; + const float y = std::sin(angle) * kRadius + kHalfCylinderHeight; + const Math::Vector3 normalBase(0.0f, std::sin(angle), 0.0f); + rings.push_back({ y, ringRadius, normalBase }); + } + + for (size_t ringIndex = 0; ringIndex < rings.size(); ++ringIndex) { + const float v = rings.size() > 1 + ? static_cast(ringIndex) / static_cast(rings.size() - 1) + : 0.0f; + for (int segment = 0; segment <= kRadialSegments; ++segment) { + const float u = static_cast(segment) / static_cast(kRadialSegments); + const float angle = u * (2.0f * kPi); + const float cosAngle = std::cos(angle); + const float sinAngle = std::sin(angle); + const Math::Vector3 radial(cosAngle, 0.0f, sinAngle); + + Math::Vector3 normal( + radial.x * rings[ringIndex].radius, + rings[ringIndex].normalBase.y * kRadius, + radial.z * rings[ringIndex].radius); + normal = normal.Normalized(); + if (normal.SqrMagnitude() <= 0.000001f) { + normal = rings[ringIndex].y >= 0.0f ? Math::Vector3::Up() : Math::Vector3::Down(); + } + + const Math::Vector3 tangent(-sinAngle, 0.0f, cosAngle); + buffers.vertices.push_back(MakeVertex( + Math::Vector3(radial.x * rings[ringIndex].radius, rings[ringIndex].y, radial.z * rings[ringIndex].radius), + normal, + tangent, + Math::Vector2(u, 1.0f - v))); + } + } + + const int stride = kRadialSegments + 1; + for (size_t ringIndex = 0; ringIndex + 1 < rings.size(); ++ringIndex) { + for (int segment = 0; segment < kRadialSegments; ++segment) { + const Core::uint32 i0 = static_cast(ringIndex * stride + static_cast(segment)); + const Core::uint32 i1 = i0 + 1; + const Core::uint32 i2 = i0 + static_cast(stride); + const Core::uint32 i3 = i2 + 1; + + buffers.indices.push_back(i0); + buffers.indices.push_back(i2); + buffers.indices.push_back(i1); + buffers.indices.push_back(i1); + buffers.indices.push_back(i2); + buffers.indices.push_back(i3); + } + } + + return buffers; +} + +Mesh* BuildMeshResource( + const Containers::String& path, + const char* displayName, + MeshBuffers&& buffers) { + if (buffers.vertices.empty() || buffers.indices.empty()) { + return nullptr; + } + + auto* mesh = new Mesh(); + IResource::ConstructParams params; + params.name = Containers::String(displayName); + params.path = path; + params.guid = ResourceGUID::Generate(path); + params.memorySize = 0; + mesh->Initialize(params); + + mesh->SetVertexData( + buffers.vertices.data(), + buffers.vertices.size() * sizeof(StaticMeshVertex), + static_cast(buffers.vertices.size()), + sizeof(StaticMeshVertex), + VertexAttribute::Position | + VertexAttribute::Normal | + VertexAttribute::Tangent | + VertexAttribute::Bitangent | + VertexAttribute::UV0); + + if (buffers.vertices.size() > 65535u) { + mesh->SetIndexData( + buffers.indices.data(), + buffers.indices.size() * sizeof(Core::uint32), + static_cast(buffers.indices.size()), + true); + } else { + std::vector compactIndices; + compactIndices.reserve(buffers.indices.size()); + for (Core::uint32 index : buffers.indices) { + compactIndices.push_back(static_cast(index)); + } + + mesh->SetIndexData( + compactIndices.data(), + compactIndices.size() * sizeof(Core::uint16), + static_cast(compactIndices.size()), + false); + } + + const Math::Bounds bounds = ComputeBounds(buffers.vertices); + mesh->SetBounds(bounds); + + MeshSection section = {}; + section.baseVertex = 0; + section.vertexCount = static_cast(buffers.vertices.size()); + section.startIndex = 0; + section.indexCount = static_cast(buffers.indices.size()); + section.materialID = 0; + section.bounds = bounds; + mesh->AddSection(section); + + return mesh; +} + +Material* BuildDefaultPrimitiveMaterial(const Containers::String& path) { + auto* material = new Material(); + IResource::ConstructParams params; + params.name = Containers::String("Default Primitive Material"); + params.path = path; + params.guid = ResourceGUID::Generate(path); + params.memorySize = 0; + material->Initialize(params); + + MaterialRenderState renderState = {}; + renderState.cullMode = MaterialCullMode::Back; + material->SetRenderState(renderState); + material->SetRenderQueue(MaterialRenderQueue::Geometry); + material->SetTexture( + Containers::String("baseColorTexture"), + ResourceManager::Get().Load(GetBuiltinDefaultPrimitiveTexturePath())); + material->RecalculateMemorySize(); + return material; +} + +Texture* BuildDefaultPrimitiveTexture(const Containers::String& path) { + static const unsigned char kTexturePixels[4] = { 214, 214, 214, 255 }; + + auto* texture = new Texture(); + IResource::ConstructParams params; + params.name = Containers::String("Default Primitive Albedo"); + params.path = path; + params.guid = ResourceGUID::Generate(path); + params.memorySize = 0; + texture->Initialize(params); + + if (!texture->Create( + 1, + 1, + 1, + 1, + TextureType::Texture2D, + TextureFormat::RGBA8_UNORM, + kTexturePixels, + sizeof(kTexturePixels))) { + delete texture; + return nullptr; + } + + return texture; +} + +} // namespace + +bool IsBuiltinResourcePath(const Containers::String& path) { + return path.StartsWith(kBuiltinPrefix); +} + +bool IsBuiltinMeshPath(const Containers::String& path) { + return path.StartsWith(kBuiltinMeshPrefix); +} + +bool IsBuiltinMaterialPath(const Containers::String& path) { + return path.StartsWith(kBuiltinMaterialPrefix); +} + +bool IsBuiltinTexturePath(const Containers::String& path) { + return path.StartsWith(kBuiltinTexturePrefix); +} + +const char* GetBuiltinPrimitiveDisplayName(BuiltinPrimitiveType primitiveType) { + switch (primitiveType) { + case BuiltinPrimitiveType::Cube: return "Cube"; + case BuiltinPrimitiveType::Sphere: return "Sphere"; + case BuiltinPrimitiveType::Capsule: return "Capsule"; + case BuiltinPrimitiveType::Cylinder: return "Cylinder"; + case BuiltinPrimitiveType::Plane: return "Plane"; + case BuiltinPrimitiveType::Quad: return "Quad"; + default: return "Primitive"; + } +} + +Containers::String GetBuiltinPrimitiveMeshPath(BuiltinPrimitiveType primitiveType) { + switch (primitiveType) { + case BuiltinPrimitiveType::Cube: return Containers::String("builtin://meshes/cube"); + case BuiltinPrimitiveType::Sphere: return Containers::String("builtin://meshes/sphere"); + case BuiltinPrimitiveType::Capsule: return Containers::String("builtin://meshes/capsule"); + case BuiltinPrimitiveType::Cylinder: return Containers::String("builtin://meshes/cylinder"); + case BuiltinPrimitiveType::Plane: return Containers::String("builtin://meshes/plane"); + case BuiltinPrimitiveType::Quad: return Containers::String("builtin://meshes/quad"); + default: return Containers::String(); + } +} + +Containers::String GetBuiltinDefaultPrimitiveMaterialPath() { + return Containers::String(kBuiltinDefaultPrimitiveMaterialPath); +} + +Containers::String GetBuiltinDefaultPrimitiveTexturePath() { + return Containers::String(kBuiltinDefaultPrimitiveTexturePath); +} + +bool TryParseBuiltinPrimitiveType(const Containers::String& path, BuiltinPrimitiveType& outPrimitiveType) { + if (path == GetBuiltinPrimitiveMeshPath(BuiltinPrimitiveType::Cube)) { + outPrimitiveType = BuiltinPrimitiveType::Cube; + return true; + } + if (path == GetBuiltinPrimitiveMeshPath(BuiltinPrimitiveType::Sphere)) { + outPrimitiveType = BuiltinPrimitiveType::Sphere; + return true; + } + if (path == GetBuiltinPrimitiveMeshPath(BuiltinPrimitiveType::Capsule)) { + outPrimitiveType = BuiltinPrimitiveType::Capsule; + return true; + } + if (path == GetBuiltinPrimitiveMeshPath(BuiltinPrimitiveType::Cylinder)) { + outPrimitiveType = BuiltinPrimitiveType::Cylinder; + return true; + } + if (path == GetBuiltinPrimitiveMeshPath(BuiltinPrimitiveType::Plane)) { + outPrimitiveType = BuiltinPrimitiveType::Plane; + return true; + } + if (path == GetBuiltinPrimitiveMeshPath(BuiltinPrimitiveType::Quad)) { + outPrimitiveType = BuiltinPrimitiveType::Quad; + return true; + } + + return false; +} + +LoadResult CreateBuiltinMeshResource(const Containers::String& path) { + BuiltinPrimitiveType primitiveType = BuiltinPrimitiveType::Cube; + if (!TryParseBuiltinPrimitiveType(path, primitiveType)) { + return LoadResult(Containers::String("Unknown builtin mesh: ") + path); + } + + MeshBuffers buffers; + switch (primitiveType) { + case BuiltinPrimitiveType::Cube: + buffers = CreateCubeMeshBuffers(); + break; + case BuiltinPrimitiveType::Sphere: + buffers = CreateUvSphereMeshBuffers(); + break; + case BuiltinPrimitiveType::Capsule: + buffers = CreateCapsuleMeshBuffers(); + break; + case BuiltinPrimitiveType::Cylinder: + buffers = CreateCylinderMeshBuffers(); + break; + case BuiltinPrimitiveType::Plane: + buffers = CreatePlaneMeshBuffers(); + break; + case BuiltinPrimitiveType::Quad: + buffers = CreateQuadMeshBuffers(); + break; + default: + return LoadResult(Containers::String("Unsupported builtin mesh: ") + path); + } + + FlipTriangleWinding(buffers); + + Mesh* mesh = BuildMeshResource(path, GetBuiltinPrimitiveDisplayName(primitiveType), std::move(buffers)); + if (mesh == nullptr) { + return LoadResult(Containers::String("Failed to create builtin mesh: ") + path); + } + + return LoadResult(mesh); +} + +LoadResult CreateBuiltinMaterialResource(const Containers::String& path) { + if (path != GetBuiltinDefaultPrimitiveMaterialPath()) { + return LoadResult(Containers::String("Unknown builtin material: ") + path); + } + + Material* material = BuildDefaultPrimitiveMaterial(path); + if (material == nullptr) { + return LoadResult(Containers::String("Failed to create builtin material: ") + path); + } + + return LoadResult(material); +} + +LoadResult CreateBuiltinTextureResource(const Containers::String& path) { + if (path != GetBuiltinDefaultPrimitiveTexturePath()) { + return LoadResult(Containers::String("Unknown builtin texture: ") + path); + } + + Texture* texture = BuildDefaultPrimitiveTexture(path); + if (texture == nullptr) { + return LoadResult(Containers::String("Failed to create builtin texture: ") + path); + } + + return LoadResult(texture); +} + +} // namespace Resources +} // namespace XCEngine diff --git a/engine/src/Resources/Material/Material.cpp b/engine/src/Resources/Material/Material.cpp index da1350df..022b96ff 100644 --- a/engine/src/Resources/Material/Material.cpp +++ b/engine/src/Resources/Material/Material.cpp @@ -46,7 +46,7 @@ void Material::SetShaderPass(const Containers::String& shaderPass) { } void Material::SetTag(const Containers::String& name, const Containers::String& value) { - for (TagEntry& tag : m_tags) { + for (MaterialTagEntry& tag : m_tags) { if (tag.name == name) { tag.value = value; UpdateMemorySize(); @@ -54,7 +54,7 @@ void Material::SetTag(const Containers::String& name, const Containers::String& } } - TagEntry tag; + MaterialTagEntry tag; tag.name = name; tag.value = value; m_tags.PushBack(tag); @@ -62,7 +62,7 @@ void Material::SetTag(const Containers::String& name, const Containers::String& } Containers::String Material::GetTag(const Containers::String& name) const { - for (const TagEntry& tag : m_tags) { + for (const MaterialTagEntry& tag : m_tags) { if (tag.name == name) { return tag.value; } @@ -72,7 +72,7 @@ Containers::String Material::GetTag(const Containers::String& name) const { } bool Material::HasTag(const Containers::String& name) const { - for (const TagEntry& tag : m_tags) { + for (const MaterialTagEntry& tag : m_tags) { if (tag.name == name) { return true; } @@ -99,6 +99,14 @@ void Material::ClearTags() { UpdateMemorySize(); } +Containers::String Material::GetTagName(Core::uint32 index) const { + return index < m_tags.Size() ? m_tags[index].name : Containers::String(); +} + +Containers::String Material::GetTagValue(Core::uint32 index) const { + return index < m_tags.Size() ? m_tags[index].value : Containers::String(); +} + void Material::SetFloat(const Containers::String& name, float value) { MaterialProperty prop; prop.name = name; @@ -180,7 +188,7 @@ void Material::SetTexture(const Containers::String& name, const ResourceHandle(m_textureBindings.Size()); binding.texture = texture; @@ -246,6 +254,24 @@ ResourceHandle Material::GetTexture(const Containers::String& name) con return ResourceHandle(); } +Containers::String Material::GetTextureBindingName(Core::uint32 index) const { + return index < m_textureBindings.Size() ? m_textureBindings[index].name : Containers::String(); +} + +ResourceHandle Material::GetTextureBindingTexture(Core::uint32 index) const { + return index < m_textureBindings.Size() ? m_textureBindings[index].texture : ResourceHandle(); +} + +std::vector Material::GetProperties() const { + std::vector properties; + const auto pairs = m_properties.GetPairs(); + properties.reserve(pairs.Size()); + for (const auto& pair : pairs) { + properties.push_back(pair.second); + } + return properties; +} + void Material::UpdateConstantBuffer() { m_constantBufferData.Clear(); UpdateMemorySize(); @@ -275,13 +301,13 @@ void Material::UpdateMemorySize() { m_memorySize = m_constantBufferData.Size() + sizeof(MaterialRenderState) + m_shaderPass.Length() + - m_tags.Size() * sizeof(TagEntry) + - m_textureBindings.Size() * sizeof(TextureBinding) + + m_tags.Size() * sizeof(MaterialTagEntry) + + m_textureBindings.Size() * sizeof(MaterialTextureBinding) + m_properties.Size() * sizeof(MaterialProperty) + m_name.Length() + m_path.Length(); - for (const TagEntry& tag : m_tags) { + for (const MaterialTagEntry& tag : m_tags) { m_memorySize += tag.name.Length(); m_memorySize += tag.value.Length(); } diff --git a/engine/src/Resources/Material/MaterialLoader.cpp b/engine/src/Resources/Material/MaterialLoader.cpp index 2a7c235f..44dd16c2 100644 --- a/engine/src/Resources/Material/MaterialLoader.cpp +++ b/engine/src/Resources/Material/MaterialLoader.cpp @@ -1,10 +1,12 @@ #include +#include #include #include #include #include #include +#include #include namespace XCEngine { @@ -531,6 +533,24 @@ ResourceHandle LoadShaderHandle(const Containers::String& shaderPath) { return ResourceHandle(static_cast(shaderResult.resource)); } +bool MaterialFileExists(const Containers::String& path) { + const std::filesystem::path inputPath(path.CStr()); + if (std::filesystem::exists(inputPath)) { + return true; + } + + if (inputPath.is_absolute()) { + return false; + } + + const Containers::String& resourceRoot = ResourceManager::Get().GetResourceRoot(); + if (resourceRoot.Empty()) { + return false; + } + + return std::filesystem::exists(std::filesystem::path(resourceRoot.CStr()) / inputPath); +} + } // namespace MaterialLoader::MaterialLoader() = default; @@ -546,6 +566,10 @@ Containers::Array MaterialLoader::GetSupportedExtensions() c } bool MaterialLoader::CanLoad(const Containers::String& path) const { + if (IsBuiltinMaterialPath(path)) { + return true; + } + Containers::String ext = GetExtension(path); return ext == "mat" || ext == "material" || ext == "json"; } @@ -553,17 +577,22 @@ bool MaterialLoader::CanLoad(const Containers::String& path) const { LoadResult MaterialLoader::Load(const Containers::String& path, const ImportSettings* settings) { (void)settings; - Containers::Array data = ReadFileData(path); - if (data.Empty()) { - return LoadResult("Failed to read material file: " + path); + if (IsBuiltinMaterialPath(path)) { + return CreateBuiltinMaterialResource(path); } + Containers::Array data = ReadFileData(path); Material* material = new Material(); material->m_path = path; material->m_name = path; material->m_guid = ResourceGUID::Generate(path); - if (!ParseMaterialData(data, material)) { + if (data.Empty() && !MaterialFileExists(path)) { + delete material; + return LoadResult("Failed to read material file: " + path); + } + + if (!data.Empty() && !ParseMaterialData(data, material)) { delete material; return LoadResult("Failed to parse material file: " + path); } diff --git a/engine/src/Resources/Mesh/MeshLoader.cpp b/engine/src/Resources/Mesh/MeshLoader.cpp index cc287323..7ee132fd 100644 --- a/engine/src/Resources/Mesh/MeshLoader.cpp +++ b/engine/src/Resources/Mesh/MeshLoader.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -14,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -507,6 +510,218 @@ void ProcessNode(const aiNode& node, } } +Containers::String ReadBinaryString(std::ifstream& stream) { + Core::uint32 length = 0; + stream.read(reinterpret_cast(&length), sizeof(length)); + if (!stream || length == 0) { + return Containers::String(); + } + + std::string buffer(length, '\0'); + stream.read(buffer.data(), length); + if (!stream) { + return Containers::String(); + } + + return Containers::String(buffer.c_str()); +} + +void ApplyMaterialProperty(Material& material, const MaterialProperty& property) { + switch (property.type) { + case MaterialPropertyType::Float: + material.SetFloat(property.name, property.value.floatValue[0]); + break; + case MaterialPropertyType::Float2: + material.SetFloat2(property.name, + Math::Vector2(property.value.floatValue[0], property.value.floatValue[1])); + break; + case MaterialPropertyType::Float3: + material.SetFloat3(property.name, + Math::Vector3(property.value.floatValue[0], property.value.floatValue[1], property.value.floatValue[2])); + break; + case MaterialPropertyType::Float4: + material.SetFloat4(property.name, + Math::Vector4(property.value.floatValue[0], + property.value.floatValue[1], + property.value.floatValue[2], + property.value.floatValue[3])); + break; + case MaterialPropertyType::Int: + material.SetInt(property.name, property.value.intValue[0]); + break; + case MaterialPropertyType::Bool: + material.SetBool(property.name, property.value.boolValue); + break; + default: + break; + } +} + +LoadResult LoadMeshArtifact(const Containers::String& path) { + std::ifstream input(path.CStr(), std::ios::binary); + if (!input.is_open()) { + return LoadResult(Containers::String("Failed to read mesh artifact: ") + path); + } + + MeshArtifactHeader header; + input.read(reinterpret_cast(&header), sizeof(header)); + if (!input) { + return LoadResult(Containers::String("Failed to parse mesh artifact header: ") + path); + } + + const std::string magic(header.magic, header.magic + 7); + if (magic != "XCMESH1") { + return LoadResult(Containers::String("Invalid mesh artifact magic: ") + path); + } + + auto mesh = std::make_unique(); + + IResource::ConstructParams params; + params.name = GetResourceNameFromPath(path); + params.path = path; + params.guid = ResourceGUID::Generate(path); + params.memorySize = 0; + mesh->Initialize(params); + + Containers::Array sections; + sections.Resize(header.sectionCount); + for (Core::uint32 index = 0; index < header.sectionCount; ++index) { + input.read(reinterpret_cast(§ions[index]), sizeof(MeshSection)); + if (!input) { + return LoadResult(Containers::String("Failed to read mesh sections: ") + path); + } + } + + Containers::Array vertexData; + vertexData.Resize(static_cast(header.vertexDataSize)); + if (header.vertexDataSize > 0) { + input.read(reinterpret_cast(vertexData.Data()), static_cast(header.vertexDataSize)); + if (!input) { + return LoadResult(Containers::String("Failed to read mesh vertex data: ") + path); + } + } + + Containers::Array indexData; + indexData.Resize(static_cast(header.indexDataSize)); + if (header.indexDataSize > 0) { + input.read(reinterpret_cast(indexData.Data()), static_cast(header.indexDataSize)); + if (!input) { + return LoadResult(Containers::String("Failed to read mesh index data: ") + path); + } + } + + mesh->SetVertexData(vertexData.Data(), + vertexData.Size(), + header.vertexCount, + header.vertexStride, + static_cast(header.vertexAttributes)); + mesh->SetIndexData(indexData.Data(), + indexData.Size(), + header.indexCount, + header.use32BitIndex != 0); + + for (const MeshSection& section : sections) { + mesh->AddSection(section); + } + + Math::Bounds bounds; + bounds.SetMinMax(header.boundsMin, header.boundsMax); + mesh->SetBounds(bounds); + + std::vector textureFiles; + textureFiles.reserve(header.textureCount); + for (Core::uint32 textureIndex = 0; textureIndex < header.textureCount; ++textureIndex) { + textureFiles.push_back(ReadBinaryString(input)); + } + + std::unordered_map loadedTextures; + TextureLoader textureLoader; + const std::filesystem::path artifactDirectory = std::filesystem::path(path.CStr()).parent_path(); + + for (Core::uint32 materialIndex = 0; materialIndex < header.materialCount; ++materialIndex) { + Core::uint32 materialPresent = 0; + input.read(reinterpret_cast(&materialPresent), sizeof(materialPresent)); + if (!input) { + return LoadResult(Containers::String("Failed to read mesh material flag: ") + path); + } + + if (materialPresent == 0) { + mesh->AddMaterial(nullptr); + continue; + } + + auto* material = new Material(); + material->m_name = ReadBinaryString(input); + material->m_path = ReadBinaryString(input); + material->m_guid = ResourceGUID::Generate(material->m_path); + material->m_isValid = true; + material->SetShaderPass(ReadBinaryString(input)); + + MaterialArtifactHeader materialHeader; + input.read(reinterpret_cast(&materialHeader), sizeof(materialHeader)); + if (!input) { + delete material; + return LoadResult(Containers::String("Failed to read material artifact header: ") + path); + } + + material->SetRenderQueue(materialHeader.renderQueue); + material->SetRenderState(materialHeader.renderState); + + for (Core::uint32 tagIndex = 0; tagIndex < materialHeader.tagCount; ++tagIndex) { + material->SetTag(ReadBinaryString(input), ReadBinaryString(input)); + } + + for (Core::uint32 propertyIndex = 0; propertyIndex < materialHeader.propertyCount; ++propertyIndex) { + MaterialProperty property; + property.name = ReadBinaryString(input); + + MaterialPropertyArtifact propertyArtifact; + input.read(reinterpret_cast(&propertyArtifact), sizeof(propertyArtifact)); + if (!input) { + delete material; + return LoadResult(Containers::String("Failed to read material property: ") + path); + } + + property.type = static_cast(propertyArtifact.propertyType); + property.value = propertyArtifact.value; + ApplyMaterialProperty(*material, property); + } + + for (Core::uint32 bindingIndex = 0; bindingIndex < materialHeader.textureBindingCount; ++bindingIndex) { + const Containers::String bindingName = ReadBinaryString(input); + const Containers::String textureFile = ReadBinaryString(input); + if (textureFile.Empty()) { + continue; + } + + const std::string textureKey(textureFile.CStr()); + Texture* texture = nullptr; + auto textureIt = loadedTextures.find(textureKey); + if (textureIt != loadedTextures.end()) { + texture = textureIt->second; + } else { + const Containers::String texturePath = + Containers::String((artifactDirectory / textureFile.CStr()).lexically_normal().string().c_str()); + LoadResult textureResult = textureLoader.Load(texturePath); + if (textureResult && textureResult.resource != nullptr) { + texture = static_cast(textureResult.resource); + loadedTextures.emplace(textureKey, texture); + mesh->AddTexture(texture); + } + } + + if (texture != nullptr) { + material->SetTexture(bindingName, ResourceHandle(texture)); + } + } + + material->RecalculateMemorySize(); + mesh->AddMaterial(material); + } + + return LoadResult(mesh.release()); +} + } // namespace MeshLoader::MeshLoader() = default; @@ -520,26 +735,45 @@ Containers::Array MeshLoader::GetSupportedExtensions() const extensions.PushBack(Containers::String("glb")); extensions.PushBack(Containers::String("dae")); extensions.PushBack(Containers::String("stl")); + extensions.PushBack(Containers::String("xcmesh")); return extensions; } bool MeshLoader::CanLoad(const Containers::String& path) const { + if (IsBuiltinMeshPath(path)) { + return true; + } + Containers::String ext = GetExtension(path).ToLower(); return ext == "fbx" || ext == "obj" || ext == "gltf" || - ext == "glb" || ext == "dae" || ext == "stl"; + ext == "glb" || ext == "dae" || ext == "stl" || + ext == "xcmesh"; } LoadResult MeshLoader::Load(const Containers::String& path, const ImportSettings* settings) { + if (IsBuiltinMeshPath(path)) { + return CreateBuiltinMeshResource(path); + } + const Containers::String ext = GetExtension(path).ToLower(); if (!CanLoad(path)) { return LoadResult(Containers::String("Unsupported mesh format: ") + ext); } - std::ifstream file(path.CStr(), std::ios::binary); + if (ext == "xcmesh") { + return LoadMeshArtifact(path); + } + + Containers::String resolvedPath = path; + if (!std::filesystem::path(path.CStr()).is_absolute()) { + resolvedPath = ResourceManager::Get().ResolvePath(path); + } + + std::ifstream file(resolvedPath.CStr(), std::ios::binary); if (!file.is_open()) { - return LoadResult(Containers::String("Failed to read file: ") + path); + return LoadResult(Containers::String("Failed to read file: ") + resolvedPath); } MeshImportSettings defaultSettings; @@ -549,7 +783,7 @@ LoadResult MeshLoader::Load(const Containers::String& path, const ImportSettings } Assimp::Importer importer; - const aiScene* scene = importer.ReadFile(path.CStr(), BuildPostProcessFlags(*resolvedSettings)); + const aiScene* scene = importer.ReadFile(resolvedPath.CStr(), BuildPostProcessFlags(*resolvedSettings)); if (scene == nullptr || scene->mRootNode == nullptr || (scene->mFlags & AI_SCENE_FLAGS_INCOMPLETE) != 0) { const char* errorText = importer.GetErrorString(); return LoadResult(Containers::String("Assimp failed to load mesh: ") + @@ -577,7 +811,7 @@ LoadResult MeshLoader::Load(const Containers::String& path, const ImportSettings auto* mesh = new Mesh(); IResource::ConstructParams params; - const std::string fileName = std::filesystem::path(path.CStr()).filename().string(); + const std::string fileName = std::filesystem::path(resolvedPath.CStr()).filename().string(); params.name = Containers::String(fileName.c_str()); params.path = path; params.guid = ResourceGUID::Generate(path); diff --git a/engine/src/Resources/Texture/TextureLoader.cpp b/engine/src/Resources/Texture/TextureLoader.cpp index 84b3e218..510042cb 100644 --- a/engine/src/Resources/Texture/TextureLoader.cpp +++ b/engine/src/Resources/Texture/TextureLoader.cpp @@ -1,7 +1,10 @@ #include +#include +#include #include #include #include +#include namespace XCEngine { namespace Resources { @@ -29,7 +32,7 @@ LoadResult CreateTextureResource(const Containers::String& path, params.name = GetResourceNameFromPath(path); params.path = path; params.guid = ResourceGUID::Generate(path); - params.memorySize = 0; + params.memorySize = pixelDataSize; texture->Initialize(params); if (!texture->Create(width, @@ -47,6 +50,40 @@ LoadResult CreateTextureResource(const Containers::String& path, return LoadResult(texture); } +LoadResult LoadTextureArtifact(const Containers::String& path) { + std::ifstream input(path.CStr(), std::ios::binary); + if (!input.is_open()) { + return LoadResult(Containers::String("Failed to read texture artifact: ") + path); + } + + TextureArtifactHeader header; + input.read(reinterpret_cast(&header), sizeof(header)); + if (!input) { + return LoadResult(Containers::String("Failed to parse texture artifact header: ") + path); + } + + const std::string magic(header.magic, header.magic + 7); + if (magic != "XCTEX01") { + return LoadResult(Containers::String("Invalid texture artifact magic: ") + path); + } + + Containers::Array pixelData; + pixelData.Resize(static_cast(header.pixelDataSize)); + if (header.pixelDataSize > 0) { + input.read(reinterpret_cast(pixelData.Data()), static_cast(header.pixelDataSize)); + if (!input) { + return LoadResult(Containers::String("Failed to read texture artifact payload: ") + path); + } + } + + return CreateTextureResource(path, + static_cast(header.textureFormat), + header.width, + header.height, + pixelData.Data(), + pixelData.Size()); +} + } // namespace TextureLoader::TextureLoader() = default; @@ -62,19 +99,28 @@ Containers::Array TextureLoader::GetSupportedExtensions() co extensions.PushBack(Containers::String("gif")); extensions.PushBack(Containers::String("hdr")); extensions.PushBack(Containers::String("dds")); + extensions.PushBack(Containers::String("xctex")); return extensions; } bool TextureLoader::CanLoad(const Containers::String& path) const { + if (IsBuiltinTexturePath(path)) { + return true; + } + Containers::String ext = GetExtension(path).ToLower(); return ext == "png" || ext == "jpg" || ext == "jpeg" || ext == "tga" || ext == "bmp" || ext == "gif" || - ext == "hdr" || ext == "dds"; + ext == "hdr" || ext == "dds" || ext == "xctex"; } LoadResult TextureLoader::Load(const Containers::String& path, const ImportSettings* settings) { (void)settings; + + if (IsBuiltinTexturePath(path)) { + return CreateBuiltinTextureResource(path); + } Containers::String ext = GetExtension(path).ToLower(); @@ -82,6 +128,10 @@ LoadResult TextureLoader::Load(const Containers::String& path, const ImportSetti return LoadResult(Containers::String("Unsupported texture format: ") + ext); } + if (ext == "xctex") { + return LoadTextureArtifact(path); + } + if (ext == "dds") { return LoadResult(Containers::String("DDS texture decoding is not implemented yet: ") + path); } diff --git a/tests/Components/test_mesh_render_components.cpp b/tests/Components/test_mesh_render_components.cpp index 3b0d7429..2c6cc360 100644 --- a/tests/Components/test_mesh_render_components.cpp +++ b/tests/Components/test_mesh_render_components.cpp @@ -4,9 +4,12 @@ #include #include #include +#include #include #include +#include +#include #include using namespace XCEngine::Components; @@ -179,4 +182,59 @@ TEST(MeshRendererComponent_Test, SetMaterialPathPreservesPathWithoutLoadedResour EXPECT_EQ(component.GetMaterial(1), nullptr); } +TEST(MeshRendererComponent_Test, SerializeAndDeserializeLoadsProjectMaterialByAssetRef) { + namespace fs = std::filesystem; + + ResourceManager& manager = ResourceManager::Get(); + manager.Initialize(); + + const fs::path projectRoot = fs::temp_directory_path() / "xc_mesh_renderer_asset_ref_test"; + const fs::path assetsDir = projectRoot / "Assets"; + const fs::path materialPath = assetsDir / "runtime.material"; + + fs::remove_all(projectRoot); + fs::create_directories(assetsDir); + + { + std::ofstream materialFile(materialPath); + ASSERT_TRUE(materialFile.is_open()); + materialFile << "{\n"; + materialFile << " \"renderQueue\": \"geometry\",\n"; + materialFile << " \"renderState\": {\n"; + materialFile << " \"cull\": \"back\"\n"; + materialFile << " }\n"; + materialFile << "}"; + } + + manager.SetResourceRoot(projectRoot.string().c_str()); + + MeshRendererComponent source; + source.SetMaterialPath(0, "Assets/runtime.material"); + + ASSERT_EQ(source.GetMaterialCount(), 1u); + ASSERT_NE(source.GetMaterial(0), nullptr); + ASSERT_EQ(source.GetMaterialPath(0), "Assets/runtime.material"); + ASSERT_EQ(source.GetMaterialAssetRefs().size(), 1u); + EXPECT_TRUE(source.GetMaterialAssetRefs()[0].IsValid()); + + std::stringstream stream; + source.Serialize(stream); + const std::string serialized = stream.str(); + EXPECT_NE(serialized.find("materialRefs="), std::string::npos); + EXPECT_EQ(serialized.find("materialRefs=;"), std::string::npos); + + std::stringstream deserializeStream(serialized); + MeshRendererComponent target; + target.Deserialize(deserializeStream); + + ASSERT_EQ(target.GetMaterialCount(), 1u); + EXPECT_EQ(target.GetMaterialPath(0), "Assets/runtime.material"); + ASSERT_NE(target.GetMaterial(0), nullptr); + EXPECT_TRUE(target.GetMaterialAssetRefs()[0].IsValid()); + + manager.SetResourceRoot(""); + manager.Shutdown(); + fs::remove_all(projectRoot); +} + } // namespace diff --git a/tests/Resources/Material/test_material_loader.cpp b/tests/Resources/Material/test_material_loader.cpp index 667eefd1..2e7c5083 100644 --- a/tests/Resources/Material/test_material_loader.cpp +++ b/tests/Resources/Material/test_material_loader.cpp @@ -141,4 +141,47 @@ TEST(MaterialLoader, RejectsUnknownRenderStateEnum) { std::remove(materialPath.string().c_str()); } +TEST(MaterialLoader, ResourceManagerLoadsRelativeMaterialFromResourceRoot) { + namespace fs = std::filesystem; + + ResourceManager& manager = ResourceManager::Get(); + manager.Initialize(); + + const fs::path previousPath = fs::current_path(); + const fs::path projectRoot = fs::temp_directory_path() / "xc_material_loader_resource_root"; + const fs::path assetsDir = projectRoot / "Assets"; + const fs::path materialPath = assetsDir / "relative.material"; + + fs::remove_all(projectRoot); + fs::create_directories(assetsDir); + + { + std::ofstream materialFile(materialPath); + ASSERT_TRUE(materialFile.is_open()); + materialFile << "{\n"; + materialFile << " \"renderQueue\": \"geometry\",\n"; + materialFile << " \"renderState\": {\n"; + materialFile << " \"cull\": \"back\",\n"; + materialFile << " \"colorWriteMask\": 15\n"; + materialFile << " }\n"; + materialFile << "}"; + } + + manager.SetResourceRoot(projectRoot.string().c_str()); + fs::current_path(projectRoot.parent_path()); + + { + const auto materialHandle = manager.Load("Assets/relative.material"); + ASSERT_TRUE(materialHandle.IsValid()); + EXPECT_EQ(materialHandle->GetRenderQueue(), static_cast(MaterialRenderQueue::Geometry)); + EXPECT_EQ(materialHandle->GetRenderState().cullMode, MaterialCullMode::Back); + EXPECT_EQ(materialHandle->GetRenderState().colorWriteMask, 15); + } + + fs::current_path(previousPath); + fs::remove_all(projectRoot); + manager.SetResourceRoot(""); + manager.Shutdown(); +} + } // namespace diff --git a/tests/Resources/Mesh/test_mesh_loader.cpp b/tests/Resources/Mesh/test_mesh_loader.cpp index 373b49fc..730aa3a4 100644 --- a/tests/Resources/Mesh/test_mesh_loader.cpp +++ b/tests/Resources/Mesh/test_mesh_loader.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,7 +7,10 @@ #include #include #include + +#include #include +#include using namespace XCEngine::Resources; using namespace XCEngine::Containers; @@ -143,4 +147,102 @@ TEST(MeshLoader, ImportsMaterialTexturesFromObj) { delete mesh; } +TEST(MeshLoader, AssetDatabaseCreatesModelArtifactAndReusesItWithoutReimport) { + namespace fs = std::filesystem; + using namespace std::chrono_literals; + + const fs::path projectRoot = fs::temp_directory_path() / "xc_mesh_library_cache_test"; + const fs::path assetsDir = projectRoot / "Assets"; + + fs::remove_all(projectRoot); + fs::create_directories(assetsDir); + fs::copy_file(GetMeshFixturePath("textured_triangle.obj"), + assetsDir / "textured_triangle.obj", + fs::copy_options::overwrite_existing); + fs::copy_file(GetMeshFixturePath("textured_triangle.mtl"), + assetsDir / "textured_triangle.mtl", + fs::copy_options::overwrite_existing); + fs::copy_file(GetMeshFixturePath("checker.bmp"), + assetsDir / "checker.bmp", + fs::copy_options::overwrite_existing); + + AssetDatabase database; + database.Initialize(projectRoot.string().c_str()); + + AssetDatabase::ResolvedAsset firstResolve; + ASSERT_TRUE(database.EnsureArtifact("Assets/textured_triangle.obj", ResourceType::Mesh, firstResolve)); + ASSERT_TRUE(firstResolve.exists); + ASSERT_TRUE(firstResolve.artifactReady); + EXPECT_TRUE(fs::exists(projectRoot / "Assets" / "textured_triangle.obj.meta")); + EXPECT_TRUE(fs::exists(projectRoot / "Library" / "SourceAssetDB" / "assets.db")); + EXPECT_TRUE(fs::exists(projectRoot / "Library" / "ArtifactDB" / "artifacts.db")); + EXPECT_TRUE(fs::exists(firstResolve.artifactMainPath.CStr())); + EXPECT_TRUE(fs::exists((fs::path(firstResolve.artifactDirectory.CStr()) / "texture_0.xctex"))); + + AssetRef assetRef; + ASSERT_TRUE(database.TryGetAssetRef("Assets/textured_triangle.obj", ResourceType::Mesh, assetRef)); + EXPECT_TRUE(assetRef.IsValid()); + + const auto originalArtifactWriteTime = fs::last_write_time(firstResolve.artifactMainPath.CStr()); + std::this_thread::sleep_for(50ms); + + AssetDatabase::ResolvedAsset secondResolve; + ASSERT_TRUE(database.EnsureArtifact("Assets/textured_triangle.obj", ResourceType::Mesh, secondResolve)); + EXPECT_EQ(firstResolve.artifactMainPath, secondResolve.artifactMainPath); + EXPECT_EQ(originalArtifactWriteTime, fs::last_write_time(secondResolve.artifactMainPath.CStr())); + + database.Shutdown(); + fs::remove_all(projectRoot); +} + +TEST(MeshLoader, ResourceManagerLoadsModelByAssetRefFromProjectAssets) { + namespace fs = std::filesystem; + + ResourceManager& manager = ResourceManager::Get(); + manager.Initialize(); + + const fs::path projectRoot = fs::temp_directory_path() / "xc_mesh_asset_ref_test"; + const fs::path assetsDir = projectRoot / "Assets"; + + fs::remove_all(projectRoot); + fs::create_directories(assetsDir); + fs::copy_file(GetMeshFixturePath("textured_triangle.obj"), + assetsDir / "textured_triangle.obj", + fs::copy_options::overwrite_existing); + fs::copy_file(GetMeshFixturePath("textured_triangle.mtl"), + assetsDir / "textured_triangle.mtl", + fs::copy_options::overwrite_existing); + fs::copy_file(GetMeshFixturePath("checker.bmp"), + assetsDir / "checker.bmp", + fs::copy_options::overwrite_existing); + + manager.SetResourceRoot(projectRoot.string().c_str()); + + const auto firstHandle = manager.Load("Assets/textured_triangle.obj"); + ASSERT_TRUE(firstHandle.IsValid()); + EXPECT_EQ(firstHandle->GetVertexCount(), 3u); + EXPECT_EQ(firstHandle->GetIndexCount(), 3u); + EXPECT_GE(firstHandle->GetMaterials().Size(), 1u); + EXPECT_EQ(firstHandle->GetTextures().Size(), 1u); + const auto initialMaterialCount = firstHandle->GetMaterials().Size(); + + AssetRef assetRef; + ASSERT_TRUE(manager.TryGetAssetRef("Assets/textured_triangle.obj", ResourceType::Mesh, assetRef)); + EXPECT_TRUE(assetRef.IsValid()); + + manager.UnloadAll(); + + const auto secondHandle = manager.Load(assetRef); + ASSERT_TRUE(secondHandle.IsValid()); + EXPECT_EQ(secondHandle->GetPath(), "Assets/textured_triangle.obj"); + EXPECT_EQ(secondHandle->GetVertexCount(), 3u); + EXPECT_EQ(secondHandle->GetIndexCount(), 3u); + EXPECT_EQ(secondHandle->GetMaterials().Size(), initialMaterialCount); + EXPECT_EQ(secondHandle->GetTextures().Size(), 1u); + + manager.SetResourceRoot(""); + manager.Shutdown(); + fs::remove_all(projectRoot); +} + } // namespace diff --git a/tests/Resources/Texture/test_texture_loader.cpp b/tests/Resources/Texture/test_texture_loader.cpp index 76530979..1b9338db 100644 --- a/tests/Resources/Texture/test_texture_loader.cpp +++ b/tests/Resources/Texture/test_texture_loader.cpp @@ -1,8 +1,15 @@ #include +#include +#include +#include #include #include #include + +#include #include +#include +#include using namespace XCEngine::Resources; using namespace XCEngine::Containers; @@ -57,4 +64,90 @@ TEST(TextureLoader, LoadValidBmpTexture) { delete texture; } +TEST(TextureLoader, AssetDatabaseCreatesTextureArtifactAndReusesItWithoutReimport) { + namespace fs = std::filesystem; + using namespace std::chrono_literals; + + const fs::path projectRoot = fs::temp_directory_path() / "xc_texture_library_cache_test"; + const fs::path assetsDir = projectRoot / "Assets"; + const fs::path texturePath = assetsDir / "checker.bmp"; + + fs::remove_all(projectRoot); + fs::create_directories(assetsDir); + fs::copy_file(GetTextureFixturePath("checker.bmp"), texturePath, fs::copy_options::overwrite_existing); + + AssetDatabase database; + database.Initialize(projectRoot.string().c_str()); + + AssetDatabase::ResolvedAsset firstResolve; + ASSERT_TRUE(database.EnsureArtifact("Assets/checker.bmp", ResourceType::Texture, firstResolve)); + ASSERT_TRUE(firstResolve.exists); + ASSERT_TRUE(firstResolve.artifactReady); + EXPECT_TRUE(fs::exists(projectRoot / "Assets" / "checker.bmp.meta")); + EXPECT_TRUE(fs::exists(projectRoot / "Library" / "SourceAssetDB" / "assets.db")); + EXPECT_TRUE(fs::exists(projectRoot / "Library" / "ArtifactDB" / "artifacts.db")); + EXPECT_TRUE(fs::exists(firstResolve.artifactMainPath.CStr())); + + std::ifstream metaFile(projectRoot / "Assets" / "checker.bmp.meta"); + ASSERT_TRUE(metaFile.is_open()); + std::string metaText((std::istreambuf_iterator(metaFile)), std::istreambuf_iterator()); + EXPECT_NE(metaText.find("guid:"), std::string::npos); + metaFile.close(); + + AssetRef assetRef; + ASSERT_TRUE(database.TryGetAssetRef("Assets/checker.bmp", ResourceType::Texture, assetRef)); + EXPECT_TRUE(assetRef.IsValid()); + + const auto originalArtifactWriteTime = fs::last_write_time(firstResolve.artifactMainPath.CStr()); + std::this_thread::sleep_for(50ms); + + AssetDatabase::ResolvedAsset secondResolve; + ASSERT_TRUE(database.EnsureArtifact("Assets/checker.bmp", ResourceType::Texture, secondResolve)); + EXPECT_EQ(firstResolve.artifactMainPath, secondResolve.artifactMainPath); + EXPECT_EQ(originalArtifactWriteTime, fs::last_write_time(secondResolve.artifactMainPath.CStr())); + + database.Shutdown(); + fs::remove_all(projectRoot); +} + +TEST(TextureLoader, ResourceManagerLoadsTextureByAssetRefFromProjectAssets) { + namespace fs = std::filesystem; + + ResourceManager& manager = ResourceManager::Get(); + manager.Initialize(); + + const fs::path projectRoot = fs::temp_directory_path() / "xc_texture_asset_ref_test"; + const fs::path assetsDir = projectRoot / "Assets"; + const fs::path texturePath = assetsDir / "checker.bmp"; + + fs::remove_all(projectRoot); + fs::create_directories(assetsDir); + fs::copy_file(GetTextureFixturePath("checker.bmp"), texturePath, fs::copy_options::overwrite_existing); + + manager.SetResourceRoot(projectRoot.string().c_str()); + + { + const auto firstHandle = manager.Load("Assets/checker.bmp"); + ASSERT_TRUE(firstHandle.IsValid()); + EXPECT_EQ(firstHandle->GetWidth(), 2u); + EXPECT_EQ(firstHandle->GetHeight(), 2u); + + AssetRef assetRef; + ASSERT_TRUE(manager.TryGetAssetRef("Assets/checker.bmp", ResourceType::Texture, assetRef)); + EXPECT_TRUE(assetRef.IsValid()); + + manager.UnloadAll(); + + const auto secondHandle = manager.Load(assetRef); + ASSERT_TRUE(secondHandle.IsValid()); + EXPECT_EQ(secondHandle->GetWidth(), 2u); + EXPECT_EQ(secondHandle->GetHeight(), 2u); + EXPECT_EQ(secondHandle->GetPath(), "Assets/checker.bmp"); + } + + manager.SetResourceRoot(""); + manager.Shutdown(); + fs::remove_all(projectRoot); +} + } // namespace