RenderingDataManager.h

Go to the documentation of this file.

#pragma once
#include "Engine/Ecs/EcsData.h"
#include "Engine/Material/MaterialTypes.h"
#include "Engine/Mesh/Vertex.h"
#include "Engine/Renderer/RenderData.h"
#include "VulkanBuffer.h"
#include "VulkanStagedBuffer.h"
#include <entt/entt.hpp>
#include <optional>
#include <unordered_map>
#include <unordered_set>
#include <vector>
 
#define object_rendering_id uint32_t
#define primitive_rendering_id uint32_t
#define meshlet_rendering_id uint32_t
 
namespace EngineCore
{
    class ApplicationContext;
    class AssetManager;
    class Engine;
    class MaterialAsset;
    class MeshComponent;
    class MeshAsset;
    class Renderer;
    class SceneManager;
    class TextureAsset;
    class Texture;
    class RenderingDataManager
    {
        friend class Engine;
    public:
        explicit RenderingDataManager( const Engine * engine, ApplicationContext* context );

        ~RenderingDataManager();
 
        // Non-copyable (RAII - owns GPU resources)
        RenderingDataManager( const RenderingDataManager& ) = delete;
        RenderingDataManager& operator=( const RenderingDataManager& ) = delete;

        explicit RenderingDataManager( ApplicationContext* context );

        void setSceneManager( SceneManager* sceneManager );

        void setAssetManager( AssetManager* assetManager );

        void setRenderer( Renderer* renderer );

        void onRenderableSpawned( MeshComponent* component );

        void onRenderableDestroyed( MeshComponent* component );

        void onMeshLoaded( MeshAsset* asset );

        void onMeshUnloaded( MeshAsset* asset );

        void onTextureLoaded( TextureAsset* textureAsset, const std::filesystem::path& texturePath );

        void onTextureUnloaded( TextureAsset* textureAsset );

        void onMaterialLoaded( MaterialAsset* materialAsset );

        void onMaterialUnloaded( MaterialAsset* materialAsset );

        [[nodiscard]] bool needsUpdate() const { return isDirty; }

        [[nodiscard]] uint64_t getDataVersion() const { return dataVersion_; }

        void clearDirty() { isDirty = false; }

        void processPendingMeshLoads();

        void processCompletedMeshletGenerations(std::vector<Ecs::CompletedMeshletGeneration>&& completions);

        bool updateIfDirty();

        bool updateTransforms();

        [[nodiscard]] std::vector<VulkanStagedBufferSyncObjects>& getPendingTransformSyncObjects() { return pendingTransformSyncObjects_; }

        void clearPendingTransformSyncObjects() { pendingTransformSyncObjects_.clear(); }
 
        // Buffer getters for compute pipeline binding (returns device-local buffer)
        [[nodiscard]] const VulkanBuffer& getPrimitiveCullingBuffer() const { return primitiveCullingData.value().getBuffer(); }
        [[nodiscard]] const VulkanBuffer& getPrimitiveMeshletBuffer() const { return primitiveMeshletData.value().getBuffer(); }
        [[nodiscard]] const VulkanBuffer& getPerObjectDataBuffer() const { return perObjectDataBuffer.value().getBuffer(); }
        [[nodiscard]] const VulkanBuffer& getPrimitiveRenderDataBuffer() const { return primitiveRenderData.value().getBuffer(); }
        [[nodiscard]] const VulkanBuffer& getLocalBoundsBuffer() const { return localBoundsBuffer.value().getBuffer(); }
        [[nodiscard]] const VulkanBuffer& getMeshletBuffer() const { return meshletBuffer.value().getBuffer(); }
        [[nodiscard]] const VulkanBuffer& getMeshletBoundsBuffer() const { return meshletBoundsData.value().getBuffer(); }
        [[nodiscard]] const VulkanBuffer& getVertexBuffer() const { return vertexBuffer.value().getBuffer(); }
        [[nodiscard]] const VulkanBuffer& getTriangleBuffer() const { return triangleBuffer.value().getBuffer(); }
 
        // Instancing buffer getters (for geometry deduplication)
        [[nodiscard]] const VulkanBuffer& getMeshGeometryDataBuffer() const { return meshGeometryDataBuffer.value().getBuffer(); }
        [[nodiscard]] const VulkanBuffer& getInstanceDataBuffer() const { return instanceDataBuffer.value().getBuffer(); }
        [[nodiscard]] const VulkanBuffer& getInstanceCullingDataBuffer() const { return instanceCullingDataBuffer.value().getBuffer(); }
 
        // Vertex shader path buffer getters (for single-meshlet geometry)
        [[nodiscard]] const VulkanBuffer& getVSIndexBuffer() const { return vsIndexBuffer_.value().getBuffer(); }
        [[nodiscard]] const VulkanBuffer& getSingleMeshletGeometryBuffer() const { return singleMeshletGeometryBuffer_.value().getBuffer(); }
        [[nodiscard]] uint32_t getSingleMeshletGeometryCount() const { return singleMeshletGeometryCount_; }
        [[nodiscard]] uint32_t getMultiMeshletGeometryCount() const { return multiMeshletGeometryCount_; }
        [[nodiscard]] bool hasVertexShaderPath() const { return singleMeshletGeometryCount_ > 0; }
 
        // LOD buffer getters (for cluster-based LOD selection)
        [[nodiscard]] const VulkanBuffer& getClusterLodDataBuffer() const { return clusterLodDataBuffer_.value().getBuffer(); }
        [[nodiscard]] const VulkanBuffer& getClusterGroupDataBuffer() const { return clusterGroupDataBuffer_.value().getBuffer(); }
        [[nodiscard]] uint32_t getClusterCount() const { return clusterCount_; }
        [[nodiscard]] uint32_t getClusterGroupCount() const { return clusterGroupCount_; }
        [[nodiscard]] bool hasLodData() const { return clusterCount_ > 0; }

        [[nodiscard]] uint32_t getUniqueGeometryCount() const { return uniqueGeometryCount_; }

        [[nodiscard]] uint32_t getInstanceCount() const { return instanceCount_; }

        [[nodiscard]] bool isInstancingEnabled() const { return instancingEnabled_; }

        void setInstancingEnabled(bool enabled) { instancingEnabled_ = enabled; markDirty(); }

        [[nodiscard]] std::vector<VulkanStagedBufferSyncObjects>& getPendingSyncObjects() { return pendingSyncObjects_; }

        void clearPendingSyncObjects() { pendingSyncObjects_.clear(); }

        void processPendingDeletions();

        [[nodiscard]] uint32_t getPrimitiveCount() const { return primitiveCount; }

        [[nodiscard]] uint32_t getMeshletCount() const { return meshletCount; }

        void collectMaterialsFromScene();

        [[nodiscard]] PipelineNames getPipelineForPrimitive(primitive_rendering_id primitiveId) const;

        void queueTextureForUpload(Texture* texture);

        [[nodiscard]] bool hasTexturesToUpload() const;

        [[nodiscard]] std::vector<Texture*> getTexturesToUpload() const;

        void clearTexturesToUpload();

        [[nodiscard]] std::vector<VkDescriptorImageInfo> generateTextureDescriptorInfos() const;

        [[nodiscard]] uint32_t getTextureCount() const;

        [[nodiscard]] const VulkanBuffer& getMaterialBufferForPipeline(PipelineNames pipelineName) const;

        void uploadMaterialBuffers();

        [[nodiscard]] bool needsMaterialUpdate() const { return materialsDirty_; }

        void markMaterialsDirty() { materialsDirty_ = true; }

        [[nodiscard]] const VulkanBuffer& getSHProbeBuffer() const;

        void initializeSHProbeBuffer();

        void initializeDefaultTextures();

        enum class DefaultTextureType { Normal, White, Black };
        [[nodiscard]] uint32_t getValidTextureIndex(uint32_t textureIndex, DefaultTextureType defaultType);
 
    private:

        bool updatePrimitiveData();

        bool updatePrimitiveDataInstanced();

        void invalidateGeometryCache();
 
        const Engine* engine = nullptr;
        ApplicationContext* context = nullptr;
 
        // Injected dependencies (used when Engine is null)
        SceneManager* injectedSceneManager = nullptr;
        AssetManager* injectedAssetManager = nullptr;
        Renderer* injectedRenderer = nullptr;
 
        // GPU buffers indexed by primitive_rendering_id (using VulkanStagedBuffer for deferred deletion)
        std::optional<VulkanStagedBuffer> primitiveCullingData;    
        std::optional<VulkanStagedBuffer> primitiveMeshletData;    
        std::optional<VulkanStagedBuffer> perObjectDataBuffer;     
        std::optional<VulkanStagedBuffer> primitiveRenderData;     
        std::optional<VulkanStagedBuffer> localBoundsBuffer;       
 
        // GPU buffers indexed by meshlet_rendering_id
        std::optional<VulkanStagedBuffer> meshletBuffer;           
        std::optional<VulkanStagedBuffer> meshletBoundsData;       
 
        // Geometry data buffers
        std::optional<VulkanStagedBuffer> vertexBuffer;            
        std::optional<VulkanStagedBuffer> triangleBuffer;          
 
        // Instancing buffers (for geometry deduplication)
        std::optional<VulkanStagedBuffer> meshGeometryDataBuffer;  
        std::optional<VulkanStagedBuffer> instanceDataBuffer;      
        std::optional<VulkanStagedBuffer> instanceCullingDataBuffer; 
 
        // Vertex shader path buffers (for single-meshlet geometry - faster than mesh shader)
        std::optional<VulkanStagedBuffer> vsIndexBuffer_;                
        std::optional<VulkanStagedBuffer> singleMeshletGeometryBuffer_;  
 
        // LOD buffers (for cluster-based LOD selection)
        std::optional<VulkanStagedBuffer> clusterLodDataBuffer_;    
        std::optional<VulkanStagedBuffer> clusterGroupDataBuffer_;  
 
        // Pending sync objects for staged buffer uploads (structural updates)
        std::vector<VulkanStagedBufferSyncObjects> pendingSyncObjects_;
 
        // Pending sync objects for transform updates (separate from structural)
        std::vector<VulkanStagedBufferSyncObjects> pendingTransformSyncObjects_;
 
        // Counts for dispatch sizing
        uint32_t primitiveCount = 0;
        uint32_t meshletCount = 0;

        bool isDirty = false;

        uint64_t dataVersion_ = 0;
 
        // Instancing state
        bool instancingEnabled_ = true;  
        bool geometryDirty_ = false;     
        bool instancesDirty_ = false;    
 
        // Instancing counts
        uint32_t uniqueGeometryCount_ = 0;  
        uint32_t instanceCount_ = 0;        
 
        // Incremental geometry update tracking
        // These persist across updates to enable append-only geometry buffers
        uint32_t nextGeometryId_ = 0;           
        meshlet_rendering_id nextMeshletId_ = 0; 
        size_t committedVertexCount_ = 0;        
        size_t committedTriangleCount_ = 0;      
        size_t committedMeshletCount_ = 0;       
        size_t committedGeometryCount_ = 0;      
        size_t committedVsIndexCount_ = 0;       
        size_t committedSingleMeshletGeoCount_ = 0; 
        bool geometryNeedsFullRebuild_ = true;   
 
        // Vertex shader path counts (single-meshlet geometry classification)
        uint32_t singleMeshletGeometryCount_ = 0;  
        uint32_t multiMeshletGeometryCount_ = 0;   
 
        // LOD counts
        uint32_t clusterCount_ = 0;        
        uint32_t clusterGroupCount_ = 0;   

        struct GeometryCacheKey
        {
            MeshAsset* meshAsset;
            uint32_t primitiveIndex;
 
            bool operator==(const GeometryCacheKey& other) const {
                return meshAsset == other.meshAsset && primitiveIndex == other.primitiveIndex;
            }
        };
 
        struct GeometryCacheKeyHash
        {
            std::size_t operator()(const GeometryCacheKey& key) const {
                std::size_t h1 = std::hash<void*>{}(static_cast<void*>(key.meshAsset));
                std::size_t h2 = std::hash<uint32_t>{}(key.primitiveIndex);
                return h1 ^ (h2 << 1);
            }
        };

        std::unordered_map<GeometryCacheKey, MeshGeometryMapping, GeometryCacheKeyHash> geometryCache_;
 
        // Material tracking
        std::unordered_map<primitive_rendering_id, PipelineNames> primitiveToPipeline;  
        std::unordered_map<PipelineNames, uint32_t> pipelinePrimitiveCounts;             
 
        // Transform dirty tracking for sparse GPU updates
        // Maps ECS entity → primitive IDs owned by that entity's MeshComponents
        std::unordered_map<entt::entity, std::vector<primitive_rendering_id>> entityToPrimitiveIds_;
        // Maps primitive ID → MeshComponent (to retrieve transform for sparse updates)
        std::unordered_map<primitive_rendering_id, MeshComponent*> primitiveIdToComponent_;
 
        // Cached transforms for incremental updates
        // Full rebuild populates this; subsequent frames only update dirty entries
        std::vector<glm::mat4> cachedTransforms_;
        // Flag indicating cachedTransforms_ needs full rebuild (set after structural changes)
        bool transformCacheValid_ = false;
 
        // Texture upload tracking
        std::vector<Texture*> texturesToUpload;  
        std::vector<Texture*> gpuTextures;       
 
        // Material buffers per pipeline type
        std::unordered_map<PipelineNames, std::optional<VulkanBuffer>> materialBuffersByPipeline;
 
        // Material tracking
        bool materialsDirty_ = true;  
        std::unordered_map<std::string, uint32_t> materialPathToFlatColorIndex_;
        std::unordered_map<std::string, uint32_t> materialPathToDiffuseIndex_;
        std::unordered_map<std::string, uint32_t> materialPathToPbrIndex_;
        std::unordered_map<std::string, uint32_t> materialPathToNormalsIndex_;
 
        // SH Probe buffer for dynamic spherical harmonic lighting
        std::optional<VulkanBuffer> shProbeBuffer_;  
        std::vector<SHProbeData> shProbeData_;       

        void markDirty();

        void initializeMaterialBuffers();

        void snapshotRenderableMeshes();

        [[nodiscard]] bool isMeshInSnapshot(MeshAsset* asset) const;
 
        // Deferred mesh loading - meshes that finish loading are queued here
        // and only acknowledged when processPendingMeshLoads() is called
        std::vector<MeshAsset*> pendingMeshLoads_;
 
        // Snapshot of render-ready meshes for the current update cycle
        // This ensures updatePrimitiveData() and updateTransforms() see the same meshes
        std::unordered_set<MeshAsset*> renderCycleMeshSnapshot_;
 
        // ============================================================================
        // Reusable working buffers (avoid per-frame heap allocations)
        // These are cleared and reused each frame instead of being locally allocated
        // ============================================================================
 
        // Geometry buffers (cleared each dirty rebuild)
        std::vector<MeshGeometryData> workingGeometryDataBuffer_;
        std::vector<UnifiedMeshlet> workingMeshletDataBuffer_;
        std::vector<MeshletBounds> workingMeshletBoundsBuffer_;
        std::vector<Vertex> workingVertexBuffer_;
        std::vector<Ecs::PackedTriangle> workingTriangleBuffer_;
 
        // Vertex shader path buffers
        std::vector<uint32_t> workingVsIndexBuffer_;
        std::vector<SingleMeshletGeometryData> workingSingleMeshletGeoData_;
 
        // LOD working buffers
        std::vector<ClusterLodData> workingClusterLodData_;
        std::vector<ClusterGroupData> workingClusterGroupData_;
 
        // Instance buffers
        std::vector<InstanceData> workingInstanceData_;
        std::vector<InstanceCullingData> workingInstanceCullingData_;
 
        // Legacy compatibility buffers
        std::vector<ObjectCullingData> workingPrimitiveCullingBuffer_;
        std::vector<LocalBoundsData> workingLocalBoundsDataBuffer_;
        std::vector<PrimitiveMeshletData> workingPrimitiveMeshletBuffer_;
        std::vector<glm::mat4> workingPerObjectData_;
        std::vector<MeshPrimitiveRenderData> workingPrimitiveRenderData_;
 
        // Material working buffers (per pipeline type, cleared on rebuild)
        std::vector<GpuDiffuseFlatColorMaterial> workingFlatColorMaterials_;
        std::vector<GpuDiffuseShaderMaterial> workingDiffuseMaterials_;
        std::vector<GpuPbrMaterial> workingPbrMaterials_;
        std::vector<GpuNormalsMaterial> workingNormalsMaterials_;
 
        // Default texture paths for PBR fallbacks (resolved via AssetManager)
        static constexpr const char* DEFAULT_NORMAL_TEXTURE_PATH = "Engine/textures/DefaultNormal.png";
        static constexpr const char* DEFAULT_WHITE_TEXTURE_PATH = "Engine/textures/DefaultWhite.png";
        static constexpr const char* DEFAULT_BLACK_TEXTURE_PATH = "Engine/textures/DefaultBlack.png";
        bool defaultTexturesLoading_ = false;  
 
    };
}