Formalize material schema and constant layout contract

2026-04-03 16:49:30 +08:00
parent 052ac28aa3
commit 03bd755e0a
10 changed files with 1841 additions and 87 deletions
--- a/engine/src/Core/Asset/AssetDatabase.cpp
+++ b/engine/src/Core/Asset/AssetDatabase.cpp
@@ -42,11 +42,42 @@ Containers::String ToContainersString(const std::string& value) {
    return Containers::String(value.c_str());
 }

+Containers::String NormalizeArtifactPathString(const fs::path& path);
+Containers::String NormalizeArtifactPathString(const Containers::String& path);
+
+void PopulateResolvedAssetResult(const Containers::String& projectRoot,
+                                 const AssetDatabase::SourceAssetRecord& sourceRecord,
+                                 const AssetDatabase::ArtifactRecord& artifactRecord,
+                                 bool imported,
+                                 AssetDatabase::ResolvedAsset& outAsset) {
+    outAsset = AssetDatabase::ResolvedAsset();
+    outAsset.exists = true;
+    outAsset.artifactReady = true;
+    outAsset.imported = imported;
+    outAsset.absolutePath =
+        ToContainersString((fs::path(projectRoot.CStr()) / sourceRecord.relativePath.CStr()).lexically_normal().generic_string());
+    outAsset.relativePath = sourceRecord.relativePath;
+    outAsset.assetGuid = sourceRecord.guid;
+    outAsset.resourceType = artifactRecord.resourceType;
+    outAsset.artifactDirectory =
+        ToContainersString((fs::path(projectRoot.CStr()) / artifactRecord.artifactDirectory.CStr()).lexically_normal().generic_string());
+    outAsset.artifactMainPath =
+        ToContainersString((fs::path(projectRoot.CStr()) / artifactRecord.mainArtifactPath.CStr()).lexically_normal().generic_string());
+    outAsset.mainLocalID = artifactRecord.mainLocalID;
+}
+
+Containers::String NormalizeArtifactPathString(const fs::path& path) {
+    if (path.empty()) {
+        return Containers::String();
+    }
+    return ToContainersString(path.lexically_normal().generic_string());
+}
+
 Containers::String NormalizeArtifactPathString(const Containers::String& path) {
    if (path.Empty()) {
        return Containers::String();
    }
-    return ToContainersString(fs::path(path.CStr()).lexically_normal().generic_string());
+    return NormalizeArtifactPathString(fs::path(path.CStr()));
 }

 bool IsProjectRelativePath(const fs::path& path) {
@@ -321,11 +352,62 @@ std::vector<MaterialProperty> GatherMaterialProperties(const Material& material)
    return material.GetProperties();
 }

+std::string EncodeAssetRef(const AssetRef& assetRef) {
+    if (!assetRef.IsValid()) {
+        return std::string();
+    }
+
+    return ToStdString(assetRef.assetGuid.ToString()) + "," +
+           std::to_string(assetRef.localID) + "," +
+           std::to_string(static_cast<int>(assetRef.resourceType));
+}
+
+AssetRef ResolveTextureBindingAssetRef(
+    const Material& material,
+    Core::uint32 bindingIndex,
+    const std::unordered_map<const Texture*, AssetRef>& textureAssetRefs,
+    const AssetDatabase* assetDatabase) {
+    const AssetRef bindingAssetRef = material.GetTextureBindingAssetRef(bindingIndex);
+    if (bindingAssetRef.IsValid()) {
+        return bindingAssetRef;
+    }
+
+    const ResourceHandle<Texture> textureHandle = material.GetTextureBindingLoadedTexture(bindingIndex);
+    const Texture* texture = textureHandle.Get();
+    if (texture != nullptr) {
+        const auto textureRefIt = textureAssetRefs.find(texture);
+        if (textureRefIt != textureAssetRefs.end()) {
+            return textureRefIt->second;
+        }
+
+        if (assetDatabase != nullptr &&
+            !texture->GetPath().Empty() &&
+            !HasVirtualPathScheme(texture->GetPath())) {
+            AssetRef resolvedAssetRef;
+            if (assetDatabase->TryGetAssetRef(texture->GetPath(), ResourceType::Texture, resolvedAssetRef)) {
+                return resolvedAssetRef;
+            }
+        }
+    }
+
+    const Containers::String bindingPath = material.GetTextureBindingPath(bindingIndex);
+    if (assetDatabase != nullptr &&
+        !bindingPath.Empty() &&
+        !HasVirtualPathScheme(bindingPath)) {
+        AssetRef resolvedAssetRef;
+        if (assetDatabase->TryGetAssetRef(bindingPath, ResourceType::Texture, resolvedAssetRef)) {
+            return resolvedAssetRef;
+        }
+    }
+
+    return AssetRef();
+}
+
 Containers::String ResolveTextureBindingPath(
    const Material& material,
    Core::uint32 bindingIndex,
    const std::unordered_map<const Texture*, Containers::String>& textureArtifactPaths) {
-    const ResourceHandle<Texture> textureHandle = material.GetTextureBindingTexture(bindingIndex);
+    const ResourceHandle<Texture> textureHandle = material.GetTextureBindingLoadedTexture(bindingIndex);
    const Texture* texture = textureHandle.Get();
    if (texture != nullptr) {
        const auto textureIt = textureArtifactPaths.find(texture);
@@ -344,7 +426,9 @@ Containers::String ResolveTextureBindingPath(
 bool WriteMaterialArtifactFile(
    const fs::path& artifactPath,
    const Material& material,
-    const std::unordered_map<const Texture*, Containers::String>& textureArtifactPaths = {}) {
+    const std::unordered_map<const Texture*, Containers::String>& textureArtifactPaths = {},
+    const std::unordered_map<const Texture*, AssetRef>& textureAssetRefs = {},
+    const AssetDatabase* assetDatabase = nullptr) {
    std::ofstream output(artifactPath, std::ios::binary | std::ios::trunc);
    if (!output.is_open()) {
        return false;
@@ -398,8 +482,11 @@ bool WriteMaterialArtifactFile(

    for (Core::uint32 bindingIndex = 0; bindingIndex < material.GetTextureBindingCount(); ++bindingIndex) {
        const Containers::String bindingName = material.GetTextureBindingName(bindingIndex);
+        const AssetRef textureAssetRef =
+            ResolveTextureBindingAssetRef(material, bindingIndex, textureAssetRefs, assetDatabase);

        WriteString(output, bindingName);
+        WriteString(output, Containers::String(EncodeAssetRef(textureAssetRef).c_str()));
        WriteString(output, ResolveTextureBindingPath(material, bindingIndex, textureArtifactPaths));
    }

@@ -564,7 +651,6 @@ void AssetDatabase::Initialize(const Containers::String& projectRoot) {
    LoadSourceAssetDB();
    LoadArtifactDB();
    ScanAssets();
-    SaveArtifactDB();
 }

 void AssetDatabase::Shutdown() {
@@ -581,8 +667,8 @@ void AssetDatabase::Shutdown() {
    m_artifactsByGuid.clear();
 }

-void AssetDatabase::Refresh() {
-    ScanAssets();
+AssetDatabase::MaintenanceStats AssetDatabase::Refresh() {
+    return ScanAssets();
 }

 bool AssetDatabase::ResolvePath(const Containers::String& requestPath,
@@ -644,6 +730,25 @@ bool AssetDatabase::TryGetAssetGuid(const Containers::String& requestPath, Asset
    return outGuid.IsValid();
 }

+bool AssetDatabase::TryGetImportableResourceType(const Containers::String& requestPath, ResourceType& outType) const {
+    outType = ResourceType::Unknown;
+
+    Containers::String absolutePath;
+    Containers::String relativePath;
+    if (!ResolvePath(requestPath, absolutePath, relativePath) || relativePath.Empty()) {
+        return false;
+    }
+
+    std::error_code ec;
+    const fs::path absoluteFsPath(absolutePath.CStr());
+    if (!fs::exists(absoluteFsPath, ec) || fs::is_directory(absoluteFsPath, ec)) {
+        return false;
+    }
+
+    outType = GetPrimaryResourceTypeForImporter(GetImporterNameForPath(relativePath, false));
+    return outType != ResourceType::Unknown;
+}
+
 bool AssetDatabase::TryGetAssetRef(const Containers::String& requestPath,
                                   ResourceType resourceType,
                                   AssetRef& outRef) const {
@@ -658,6 +763,115 @@ bool AssetDatabase::TryGetAssetRef(const Containers::String& requestPath,
    return true;
 }

+bool AssetDatabase::ReimportAsset(const Containers::String& requestPath,
+                                  ResolvedAsset& outAsset,
+                                  MaintenanceStats* outStats) {
+    outAsset = ResolvedAsset();
+    if (outStats != nullptr) {
+        *outStats = MaintenanceStats();
+    }
+
+    Containers::String absolutePath;
+    Containers::String relativePath;
+    if (!ResolvePath(requestPath, absolutePath, relativePath) || relativePath.Empty()) {
+        return false;
+    }
+
+    const fs::path absoluteFsPath(absolutePath.CStr());
+    if (!fs::exists(absoluteFsPath) || fs::is_directory(absoluteFsPath)) {
+        return false;
+    }
+
+    SourceAssetRecord sourceRecord;
+    if (!EnsureMetaForPath(absoluteFsPath, false, sourceRecord)) {
+        return false;
+    }
+
+    const ResourceType primaryType = GetPrimaryResourceTypeForImporter(sourceRecord.importerName);
+    if (primaryType == ResourceType::Unknown) {
+        return false;
+    }
+
+    ArtifactRecord rebuiltRecord;
+    if (!ImportAsset(sourceRecord, rebuiltRecord)) {
+        return false;
+    }
+
+    m_artifactsByGuid[sourceRecord.guid] = rebuiltRecord;
+    m_sourcesByGuid[sourceRecord.guid].lastKnownArtifactKey = rebuiltRecord.artifactKey;
+    m_sourcesByPathKey[ToStdString(MakeKey(sourceRecord.relativePath))].lastKnownArtifactKey = rebuiltRecord.artifactKey;
+    SaveArtifactDB();
+    SaveSourceAssetDB();
+
+    MaintenanceStats localStats;
+    localStats.importedAssetCount = 1;
+    localStats.removedArtifactCount = CleanupOrphanedArtifacts();
+    if (outStats != nullptr) {
+        *outStats = localStats;
+    }
+
+    PopulateResolvedAssetResult(m_projectRoot, sourceRecord, rebuiltRecord, true, outAsset);
+    return true;
+}
+
+bool AssetDatabase::ReimportAllAssets(MaintenanceStats* outStats) {
+    if (outStats != nullptr) {
+        *outStats = MaintenanceStats();
+    }
+
+    std::vector<SourceAssetRecord> importableRecords;
+    importableRecords.reserve(m_sourcesByGuid.size());
+
+    for (const auto& [guid, record] : m_sourcesByGuid) {
+        (void)guid;
+        if (record.isFolder) {
+            continue;
+        }
+
+        const ResourceType primaryType = GetPrimaryResourceTypeForImporter(record.importerName);
+        if (primaryType == ResourceType::Unknown) {
+            continue;
+        }
+
+        const fs::path sourcePath = fs::path(m_projectRoot.CStr()) / record.relativePath.CStr();
+        if (!fs::exists(sourcePath) || fs::is_directory(sourcePath)) {
+            continue;
+        }
+
+        importableRecords.push_back(record);
+    }
+
+    std::sort(importableRecords.begin(), importableRecords.end(), [](const SourceAssetRecord& lhs, const SourceAssetRecord& rhs) {
+        return ToStdString(lhs.relativePath) < ToStdString(rhs.relativePath);
+    });
+
+    bool allSucceeded = true;
+    MaintenanceStats localStats;
+    for (const SourceAssetRecord& record : importableRecords) {
+        ArtifactRecord rebuiltRecord;
+        if (!ImportAsset(record, rebuiltRecord)) {
+            Debug::Logger::Get().Error(
+                Debug::LogCategory::FileSystem,
+                Containers::String("[AssetDatabase] ReimportAllAssets failed path=") + record.relativePath);
+            allSucceeded = false;
+            continue;
+        }
+
+        m_artifactsByGuid[record.guid] = rebuiltRecord;
+        m_sourcesByGuid[record.guid].lastKnownArtifactKey = rebuiltRecord.artifactKey;
+        m_sourcesByPathKey[ToStdString(MakeKey(record.relativePath))].lastKnownArtifactKey = rebuiltRecord.artifactKey;
+        ++localStats.importedAssetCount;
+    }
+
+    SaveArtifactDB();
+    SaveSourceAssetDB();
+    localStats.removedArtifactCount = CleanupOrphanedArtifacts();
+    if (outStats != nullptr) {
+        *outStats = localStats;
+    }
+    return allSucceeded;
+}
+
 bool AssetDatabase::TryGetPrimaryAssetPath(const AssetGUID& guid, Containers::String& outRelativePath) const {
    const auto sourceIt = m_sourcesByGuid.find(guid);
    if (sourceIt == m_sourcesByGuid.end()) {
@@ -848,14 +1062,18 @@ void AssetDatabase::SaveArtifactDB() const {
    }
 }

-void AssetDatabase::ScanAssets() {
+AssetDatabase::MaintenanceStats AssetDatabase::ScanAssets() {
+    MaintenanceStats stats;
    std::unordered_map<std::string, bool> seenPaths;
    const fs::path assetsRootPath(m_assetsRoot.CStr());
    if (fs::exists(assetsRootPath)) {
        ScanAssetPath(assetsRootPath, seenPaths);
    }
    RemoveMissingRecords(seenPaths);
+    stats.removedArtifactCount = CleanupOrphanedArtifacts();
    SaveSourceAssetDB();
+    SaveArtifactDB();
+    return stats;
 }

 void AssetDatabase::ScanAssetPath(const fs::path& path,
@@ -902,9 +1120,75 @@ void AssetDatabase::RemoveMissingRecords(const std::unordered_map<std::string, b
        m_sourcesByPathKey.erase(recordIt);
    }

-    if (!missingPathKeys.empty()) {
-        SaveArtifactDB();
+}
+
+Core::uint32 AssetDatabase::CleanupOrphanedArtifacts() const {
+    std::error_code ec;
+    const fs::path artifactsRoot = fs::path(m_libraryRoot.CStr()) / "Artifacts";
+    if (!fs::exists(artifactsRoot, ec) || !fs::is_directory(artifactsRoot, ec)) {
+        return 0;
    }
+
+    std::unordered_set<std::string> retainedArtifactPathKeys;
+    retainedArtifactPathKeys.reserve(m_artifactsByGuid.size());
+    for (const auto& [guid, record] : m_artifactsByGuid) {
+        (void)guid;
+        if (!record.artifactDirectory.Empty()) {
+            retainedArtifactPathKeys.insert(ToStdString(MakeKey(record.artifactDirectory)));
+        }
+    }
+
+    Core::uint32 removedArtifactCount = 0;
+    for (const auto& shardEntry : fs::directory_iterator(artifactsRoot, ec)) {
+        if (ec) {
+            ec.clear();
+            break;
+        }
+
+        const fs::path shardPath = shardEntry.path();
+        if (!shardEntry.is_directory()) {
+            fs::remove_all(shardPath, ec);
+            if (!ec) {
+                ++removedArtifactCount;
+            }
+            ec.clear();
+            continue;
+        }
+
+        for (const auto& artifactEntry : fs::directory_iterator(shardPath, ec)) {
+            if (ec) {
+                ec.clear();
+                break;
+            }
+
+            const Containers::String relativeArtifactPath = NormalizeRelativePath(artifactEntry.path());
+            const std::string artifactPathKey = ToStdString(MakeKey(relativeArtifactPath));
+            if (!relativeArtifactPath.Empty() &&
+                retainedArtifactPathKeys.find(artifactPathKey) != retainedArtifactPathKeys.end()) {
+                continue;
+            }
+
+            fs::remove_all(artifactEntry.path(), ec);
+            if (!ec) {
+                ++removedArtifactCount;
+            }
+            ec.clear();
+        }
+
+        std::error_code shardEc;
+        if (fs::is_empty(shardPath, shardEc)) {
+            fs::remove(shardPath, shardEc);
+        }
+    }
+
+    if (removedArtifactCount > 0) {
+        Debug::Logger::Get().Info(
+            Debug::LogCategory::FileSystem,
+            Containers::String("[AssetDatabase] Removed orphan artifact entries count=") +
+            Containers::String(std::to_string(removedArtifactCount).c_str()));
+    }
+
+    return removedArtifactCount;
 }

 bool AssetDatabase::EnsureMetaForPath(const fs::path& sourcePath,
@@ -1228,7 +1512,9 @@ bool AssetDatabase::EnsureArtifact(const Containers::String& requestPath,
        m_sourcesByPathKey[ToStdString(MakeKey(sourceRecord.relativePath))].lastKnownArtifactKey = rebuiltRecord.artifactKey;
        SaveArtifactDB();
        SaveSourceAssetDB();
+        CleanupOrphanedArtifacts();
        artifactRecord = &m_artifactsByGuid[sourceRecord.guid];
+        outAsset.imported = true;
    }

    if (artifactRecord == nullptr) {
@@ -1236,14 +1522,7 @@ bool AssetDatabase::EnsureArtifact(const Containers::String& requestPath,
    }

    outAsset.exists = true;
-    outAsset.artifactReady = true;
-    outAsset.absolutePath = absolutePath;
-    outAsset.relativePath = sourceRecord.relativePath;
-    outAsset.assetGuid = sourceRecord.guid;
-    outAsset.resourceType = artifactRecord->resourceType;
-    outAsset.artifactDirectory = NormalizePathString(fs::path(m_projectRoot.CStr()) / artifactRecord->artifactDirectory.CStr());
-    outAsset.artifactMainPath = NormalizePathString(fs::path(m_projectRoot.CStr()) / artifactRecord->mainArtifactPath.CStr());
-    outAsset.mainLocalID = artifactRecord->mainLocalID;
+    PopulateResolvedAssetResult(m_projectRoot, sourceRecord, *artifactRecord, outAsset.imported, outAsset);

    if (ShouldTraceAssetPath(requestPath)) {
        Debug::Logger::Get().Info(
@@ -1331,7 +1610,7 @@ bool AssetDatabase::ImportMaterialAsset(const SourceAssetRecord& sourceRecord,
    }

    const bool writeOk =
-        WriteMaterialArtifactFile(fs::path(m_projectRoot.CStr()) / mainArtifactPath.CStr(), *material);
+        WriteMaterialArtifactFile(fs::path(m_projectRoot.CStr()) / mainArtifactPath.CStr(), *material, {}, {}, this);
    delete material;
    if (!writeOk) {
        return false;
@@ -1380,6 +1659,7 @@ bool AssetDatabase::ImportModelAsset(const SourceAssetRecord& sourceRecord,

    bool writeOk = true;
    std::unordered_map<const Texture*, Containers::String> textureArtifactPaths;
+    std::unordered_map<const Texture*, AssetRef> textureAssetRefs;
    for (size_t textureIndex = 0; writeOk && textureIndex < mesh->GetTextures().Size(); ++textureIndex) {
        Texture* texture = mesh->GetTextures()[textureIndex];
        if (texture == nullptr) {
@@ -1396,6 +1676,13 @@ bool AssetDatabase::ImportModelAsset(const SourceAssetRecord& sourceRecord,
        }

        textureArtifactPaths.emplace(texture, textureArtifactPath);
+
+        if (!texture->GetPath().Empty()) {
+            AssetRef textureAssetRef;
+            if (TryGetAssetRef(texture->GetPath(), ResourceType::Texture, textureAssetRef)) {
+                textureAssetRefs.emplace(texture, textureAssetRef);
+            }
+        }
    }

    std::vector<Containers::String> materialArtifactPaths;
@@ -1412,7 +1699,9 @@ bool AssetDatabase::ImportModelAsset(const SourceAssetRecord& sourceRecord,
        writeOk = WriteMaterialArtifactFile(
            fs::path(m_projectRoot.CStr()) / materialArtifactPath.CStr(),
            *material,
-            textureArtifactPaths);
+            textureArtifactPaths,
+            textureAssetRefs,
+            this);
        if (!writeOk) {
            break;
        }