// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later #pragma once #include <array> #include <atomic> #include <bit> #include <deque> #include <memory> #include <mutex> #include "common/common_types.h" #include "common/range_mutex.h" #include "common/scratch_buffer.h" #include "common/virtual_buffer.h" namespace Core { constexpr size_t DEVICE_PAGEBITS = 12ULL; constexpr size_t DEVICE_PAGESIZE = 1ULL << DEVICE_PAGEBITS; constexpr size_t DEVICE_PAGEMASK = DEVICE_PAGESIZE - 1ULL; class DeviceMemory; namespace Memory { class Memory; } template <typename DTraits> struct DeviceMemoryManagerAllocator; struct Asid { size_t id; }; template <typename Traits> class DeviceMemoryManager { using DeviceInterface = typename Traits::DeviceInterface; using DeviceMethods = typename Traits::DeviceMethods; public: DeviceMemoryManager(const DeviceMemory& device_memory); ~DeviceMemoryManager(); void BindInterface(DeviceInterface* device_inter); DAddr Allocate(size_t size); void AllocateFixed(DAddr start, size_t size); void Free(DAddr start, size_t size); void Map(DAddr address, VAddr virtual_address, size_t size, Asid asid, bool track = false); void Unmap(DAddr address, size_t size); void TrackContinuityImpl(DAddr address, VAddr virtual_address, size_t size, Asid asid); void TrackContinuity(DAddr address, VAddr virtual_address, size_t size, Asid asid) { std::scoped_lock lk(mapping_guard); TrackContinuityImpl(address, virtual_address, size, asid); } // Write / Read template <typename T> T* GetPointer(DAddr address); template <typename T> const T* GetPointer(DAddr address) const; template <typename Func> void ApplyOpOnPAddr(PAddr address, Common::ScratchBuffer<u32>& buffer, Func&& operation) { DAddr subbits = static_cast<DAddr>(address & page_mask); const u32 base = compressed_device_addr[(address >> page_bits)]; if ((base >> MULTI_FLAG_BITS) == 0) [[likely]] { const DAddr d_address = (static_cast<DAddr>(base) << page_bits) + subbits; operation(d_address); return; } InnerGatherDeviceAddresses(buffer, address); for (u32 value : buffer) { operation((static_cast<DAddr>(value) << page_bits) + subbits); } } template <typename Func> void ApplyOpOnPointer(const u8* p, Common::ScratchBuffer<u32>& buffer, Func&& operation) { PAddr address = GetRawPhysicalAddr<u8>(p); ApplyOpOnPAddr(address, buffer, operation); } PAddr GetPhysicalRawAddressFromDAddr(DAddr address) const { PAddr subbits = static_cast<PAddr>(address & page_mask); auto paddr = compressed_physical_ptr[(address >> page_bits)]; if (paddr == 0) { return 0; } return (static_cast<PAddr>(paddr - 1) << page_bits) + subbits; } template <typename T> void Write(DAddr address, T value); template <typename T> T Read(DAddr address) const; u8* GetSpan(const DAddr src_addr, const std::size_t size); const u8* GetSpan(const DAddr src_addr, const std::size_t size) const; void ReadBlock(DAddr address, void* dest_pointer, size_t size); void ReadBlockUnsafe(DAddr address, void* dest_pointer, size_t size); void WriteBlock(DAddr address, const void* src_pointer, size_t size); void WriteBlockUnsafe(DAddr address, const void* src_pointer, size_t size); Asid RegisterProcess(Memory::Memory* memory); void UnregisterProcess(Asid id); void UpdatePagesCachedCount(DAddr addr, size_t size, s32 delta); static constexpr size_t AS_BITS = Traits::device_virtual_bits; private: static constexpr size_t device_virtual_bits = Traits::device_virtual_bits; static constexpr size_t device_as_size = 1ULL << device_virtual_bits; static constexpr size_t physical_min_bits = 32; static constexpr size_t physical_max_bits = 33; static constexpr size_t page_bits = 12; static constexpr size_t page_size = 1ULL << page_bits; static constexpr size_t page_mask = page_size - 1ULL; static constexpr u32 physical_address_base = 1U << page_bits; static constexpr u32 MULTI_FLAG_BITS = 31; static constexpr u32 MULTI_FLAG = 1U << MULTI_FLAG_BITS; static constexpr u32 MULTI_MASK = ~MULTI_FLAG; template <typename T> T* GetPointerFromRaw(PAddr addr) { return reinterpret_cast<T*>(physical_base + addr); } template <typename T> const T* GetPointerFromRaw(PAddr addr) const { return reinterpret_cast<T*>(physical_base + addr); } template <typename T> PAddr GetRawPhysicalAddr(const T* ptr) const { return static_cast<PAddr>(reinterpret_cast<uintptr_t>(ptr) - physical_base); } void WalkBlock(const DAddr addr, const std::size_t size, auto on_unmapped, auto on_memory, auto increment); void InnerGatherDeviceAddresses(Common::ScratchBuffer<u32>& buffer, PAddr address); std::unique_ptr<DeviceMemoryManagerAllocator<Traits>> impl; const uintptr_t physical_base; DeviceInterface* device_inter; Common::VirtualBuffer<u32> compressed_physical_ptr; Common::VirtualBuffer<u32> compressed_device_addr; Common::VirtualBuffer<u32> continuity_tracker; // Process memory interfaces std::deque<size_t> id_pool; std::deque<Memory::Memory*> registered_processes; // Memory protection management static constexpr size_t guest_max_as_bits = 39; static constexpr size_t guest_as_size = 1ULL << guest_max_as_bits; static constexpr size_t guest_mask = guest_as_size - 1ULL; static constexpr size_t asid_start_bit = guest_max_as_bits; std::pair<Asid, VAddr> ExtractCPUBacking(size_t page_index) { auto content = cpu_backing_address[page_index]; const VAddr address = content & guest_mask; const Asid asid{static_cast<size_t>(content >> asid_start_bit)}; return std::make_pair(asid, address); } void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) { cpu_backing_address[page_index] = address | (asid.id << asid_start_bit); } Common::VirtualBuffer<VAddr> cpu_backing_address; using CounterType = u8; using CounterAtomicType = std::atomic_uint8_t; static constexpr size_t subentries = 8 / sizeof(CounterType); static constexpr size_t subentries_mask = subentries - 1; static constexpr size_t subentries_shift = std::countr_zero(sizeof(u64)) - std::countr_zero(sizeof(CounterType)); class CounterEntry final { public: CounterEntry() = default; CounterAtomicType& Count(std::size_t page) { return values[page & subentries_mask]; } const CounterAtomicType& Count(std::size_t page) const { return values[page & subentries_mask]; } private: std::array<CounterAtomicType, subentries> values{}; }; static_assert(sizeof(CounterEntry) == subentries * sizeof(CounterType), "CounterEntry should be 8 bytes!"); static constexpr size_t num_counter_entries = (1ULL << (device_virtual_bits - page_bits)) / subentries; using CachedPages = std::array<CounterEntry, num_counter_entries>; std::unique_ptr<CachedPages> cached_pages; Common::RangeMutex counter_guard; std::mutex mapping_guard; }; } // namespace Core