diff --git a/libraries/libstratosphere/include/stratosphere/fssystem.hpp b/libraries/libstratosphere/include/stratosphere/fssystem.hpp index 1016673..d870584 100644 --- a/libraries/libstratosphere/include/stratosphere/fssystem.hpp +++ b/libraries/libstratosphere/include/stratosphere/fssystem.hpp @@ -13,10 +13,10 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ - #pragma once #include #include +#include #include #include #include @@ -46,8 +46,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include diff --git a/libraries/libstratosphere/include/stratosphere/fssystem/dbm/fssystem_dbm_utils.hpp b/libraries/libstratosphere/include/stratosphere/fssystem/dbm/fssystem_dbm_utils.hpp deleted file mode 100644 index fe7bfe8..0000000 --- a/libraries/libstratosphere/include/stratosphere/fssystem/dbm/fssystem_dbm_utils.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#pragma once -#include - -namespace ams::fssystem::dbm { - - namespace { - - constexpr inline s32 CountLeadingZeros(u32 val) { - return util::CountLeadingZeros(val); - } - - constexpr inline s32 CountLeadingOnes(u32 val) { - return CountLeadingZeros(~val); - } - - inline u32 ReadU32(const u8 *buf, size_t index) { - u32 val; - std::memcpy(std::addressof(val), buf + index, sizeof(u32)); - return val; - } - - inline void WriteU32(u8 *buf, size_t index, u32 val) { - std::memcpy(buf + index, std::addressof(val), sizeof(u32)); - } - - } - -} diff --git a/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_bitmap_utils.hpp b/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_bitmap_utils.hpp new file mode 100644 index 0000000..23ca559 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_bitmap_utils.hpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +namespace ams::fssystem { + + constexpr inline s32 CountLeadingZeros(u32 val) { + return util::CountLeadingZeros(val); + } + + constexpr inline s32 CountLeadingOnes(u32 val) { + return CountLeadingZeros(~val); + } + + inline u32 ReadU32(const u8 *buf, size_t index) { + u32 val; + std::memcpy(std::addressof(val), buf + index, sizeof(u32)); + return val; + } + + inline void WriteU32(u8 *buf, size_t index, u32 val) { + std::memcpy(buf + index, std::addressof(val), sizeof(u32)); + } + + constexpr inline bool IsPowerOfTwo(s32 val) { + return util::IsPowerOfTwo(val); + } + + constexpr inline u32 ILog2(u32 val) { + AMS_ASSERT(val > 0); + return (BITSIZEOF(u32) - 1 - util::CountLeadingZeros(val)); + } + + constexpr inline u32 CeilingPowerOfTwo(u32 val) { + if (val == 0) { + return 1; + } + + return util::CeilingPowerOfTwo(val); + } + +} diff --git a/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_block_cache_buffered_storage.hpp b/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_block_cache_buffered_storage.hpp new file mode 100644 index 0000000..2bdd815 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_block_cache_buffered_storage.hpp @@ -0,0 +1,179 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace ams::fssystem { + + constexpr inline size_t IntegrityMinLayerCount = 2; + constexpr inline size_t IntegrityMaxLayerCount = 7; + constexpr inline size_t IntegrityLayerCountSave = 5; + constexpr inline size_t IntegrityLayerCountSaveDataMeta = 4; + + struct FileSystemBufferManagerSet { + fs::IBufferManager *buffers[IntegrityMaxLayerCount]; + }; + static_assert(util::is_pod::value); + + class BlockCacheBufferedStorage : public ::ams::fs::IStorage { + NON_COPYABLE(BlockCacheBufferedStorage); + NON_MOVEABLE(BlockCacheBufferedStorage); + public: + static constexpr size_t DefaultMaxCacheEntryCount = 24; + private: + using MemoryRange = fs::IBufferManager::MemoryRange; + + struct AccessRange { + s64 offset; + size_t size; + + s64 GetEndOffset() const { + return this->offset + this->size; + } + + bool IsIncluded(s64 ofs) const { + return this->offset <= ofs && ofs < this->GetEndOffset(); + } + }; + static_assert(util::is_pod::value); + + struct CacheEntry { + AccessRange range; + bool is_valid; + bool is_write_back; + bool is_cached; + bool is_flushing; + u16 lru_counter; + fs::IBufferManager::CacheHandle handle; + uintptr_t memory_address; + size_t memory_size; + + void Invalidate() { + this->is_write_back = false; + this->is_flushing = false; + } + + bool IsAllocated() const { + return this->is_valid && (this->is_write_back ? this->memory_address != 0 : this->handle != 0); + } + + bool IsWriteBack() const { + return this->is_write_back; + } + }; + static_assert(util::is_pod::value); + + using BlockCacheManager = ::ams::fssystem::impl::BlockCacheManager; + using CacheIndex = BlockCacheManager::CacheIndex; + + enum Flag : s32 { + Flag_KeepBurstMode = (1 << 8), + Flag_RealData = (1 << 10), + }; + private: + os::SdkRecursiveMutex *m_mutex; + IStorage *m_data_storage; + Result m_last_result; + s64 m_data_size; + size_t m_verification_block_size; + size_t m_verification_block_shift; + s32 m_flags; + s32 m_buffer_level; + fs::StorageType m_storage_type; + BlockCacheManager m_block_cache_manager; + public: + BlockCacheBufferedStorage(); + virtual ~BlockCacheBufferedStorage() override; + + Result Initialize(fs::IBufferManager *bm, os::SdkRecursiveMutex *mtx, IStorage *data, s64 data_size, size_t verif_block_size, s32 max_cache_entries, bool is_real_data, s8 buffer_level, bool is_keep_burst_mode, fs::StorageType storage_type); + void Finalize(); + + virtual Result Read(s64 offset, void *buffer, size_t size) override; + virtual Result Write(s64 offset, const void *buffer, size_t size) override; + + virtual Result SetSize(s64) override { R_THROW(fs::ResultUnsupportedOperationInBlockCacheBufferedStorageA()); } + virtual Result GetSize(s64 *out) override; + + virtual Result Flush() override; + + virtual Result OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) override; + using IStorage::OperateRange; + + Result Commit(); + Result OnRollback(); + + bool IsEnabledKeepBurstMode() const { + return (m_flags & Flag_KeepBurstMode) != 0; + } + + bool IsRealDataCache() const { + return (m_flags & Flag_RealData) != 0; + } + + void SetKeepBurstMode(bool en) { + if (en) { + m_flags |= Flag_KeepBurstMode; + } else { + m_flags &= ~Flag_KeepBurstMode; + } + } + + void SetRealDataCache(bool en) { + if (en) { + m_flags |= Flag_RealData; + } else { + m_flags &= ~Flag_RealData; + } + } + private: + Result FillZeroImpl(s64 offset, s64 size); + Result DestroySignatureImpl(s64 offset, s64 size); + Result InvalidateImpl(); + Result QueryRangeImpl(void *dst, size_t dst_size, s64 offset, s64 size); + + Result GetAssociateBuffer(MemoryRange *out_range, CacheEntry *out_entry, s64 offset, size_t ideal_size, bool is_allocate_for_write); + + Result StoreOrDestroyBuffer(CacheIndex *out, const MemoryRange &range, CacheEntry *entry); + + Result StoreOrDestroyBuffer(const MemoryRange &range, CacheEntry *entry) { + AMS_ASSERT(entry != nullptr); + + CacheIndex dummy; + R_RETURN(this->StoreOrDestroyBuffer(std::addressof(dummy), range, entry)); + } + + Result FlushCacheEntry(CacheIndex index, bool invalidate); + Result FlushRangeCacheEntries(s64 offset, s64 size, bool invalidate); + + Result FlushAllCacheEntries(); + Result InvalidateAllCacheEntries(); + Result ControlDirtiness(); + + Result UpdateLastResult(Result result); + + Result ReadHeadCache(MemoryRange *out_range, CacheEntry *out_entry, bool *out_cache_needed, s64 *offset, s64 *aligned_offset, s64 aligned_offset_end, char **buffer, size_t *size); + Result ReadTailCache(MemoryRange *out_range, CacheEntry *out_entry, bool *out_cache_needed, s64 offset, s64 aligned_offset, s64 *aligned_offset_end, char *buffer, size_t *size); + + Result BulkRead(s64 offset, void *buffer, size_t size, MemoryRange *range_head, MemoryRange *range_tail, CacheEntry *entry_head, CacheEntry *entry_tail, bool head_cache_needed, bool tail_cache_needed); + }; + +} diff --git a/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_buffered_storage.hpp b/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_buffered_storage.hpp new file mode 100644 index 0000000..4288f9d --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_buffered_storage.hpp @@ -0,0 +1,80 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include +#include +#include + +namespace ams::fssystem { + + class BufferedStorage : public ::ams::fs::IStorage { + NON_COPYABLE(BufferedStorage); + NON_MOVEABLE(BufferedStorage); + private: + class Cache; + class UniqueCache; + class SharedCache; + private: + fs::SubStorage m_base_storage; + fs::IBufferManager *m_buffer_manager; + size_t m_block_size; + s64 m_base_storage_size; + std::unique_ptr m_caches; + s32 m_cache_count; + Cache *m_next_acquire_cache; + Cache *m_next_fetch_cache; + os::SdkMutex m_mutex; + bool m_bulk_read_enabled; + public: + BufferedStorage(); + virtual ~BufferedStorage(); + + Result Initialize(fs::SubStorage base_storage, fs::IBufferManager *buffer_manager, size_t block_size, s32 buffer_count); + void Finalize(); + + bool IsInitialized() const { return m_caches != nullptr; } + + virtual Result Read(s64 offset, void *buffer, size_t size) override; + virtual Result Write(s64 offset, const void *buffer, size_t size) override; + + virtual Result GetSize(s64 *out) override; + virtual Result SetSize(s64 size) override; + + virtual Result Flush() override; + virtual Result OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) override; + using IStorage::OperateRange; + + void InvalidateCaches(); + + fs::IBufferManager *GetBufferManager() const { return m_buffer_manager; } + + void EnableBulkRead() { m_bulk_read_enabled = true; } + private: + Result PrepareAllocation(); + Result ControlDirtiness(); + Result ReadCore(s64 offset, void *buffer, size_t size); + + bool ReadHeadCache(s64 *offset, void *buffer, size_t *size, s64 *buffer_offset); + bool ReadTailCache(s64 offset, void *buffer, size_t *size, s64 buffer_offset); + + Result BulkRead(s64 offset, void *buffer, size_t size, bool head_cache_needed, bool tail_cache_needed); + + Result WriteCore(s64 offset, const void *buffer, size_t size); + }; + +} diff --git a/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_hierarchical_integrity_verification_storage.hpp b/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_hierarchical_integrity_verification_storage.hpp new file mode 100644 index 0000000..ea45400 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_hierarchical_integrity_verification_storage.hpp @@ -0,0 +1,204 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace ams::fssystem { + + struct HierarchicalIntegrityVerificationLevelInformation { + fs::Int64 offset; + fs::Int64 size; + s32 block_order; + u8 reserved[4]; + }; + static_assert(util::is_pod::value); + static_assert(sizeof(HierarchicalIntegrityVerificationLevelInformation) == 0x18); + static_assert(alignof(HierarchicalIntegrityVerificationLevelInformation) == 0x4); + + struct HierarchicalIntegrityVerificationInformation { + u32 max_layers; + HierarchicalIntegrityVerificationLevelInformation info[IntegrityMaxLayerCount - 1]; + fs::HashSalt seed; + + s64 GetLayeredHashSize() const { + return this->info[this->max_layers - 2].offset; + } + + s64 GetDataOffset() const { + return this->info[this->max_layers - 2].offset; + } + + s64 GetDataSize() const { + return this->info[this->max_layers - 2].size; + } + }; + static_assert(util::is_pod::value); + + struct HierarchicalIntegrityVerificationMetaInformation { + u32 magic; + u32 version; + u32 master_hash_size; + HierarchicalIntegrityVerificationInformation level_hash_info; + + /* TODO: Format */ + }; + static_assert(util::is_pod::value); + + struct HierarchicalIntegrityVerificationSizeSet { + s64 control_size; + s64 master_hash_size; + s64 layered_hash_sizes[IntegrityMaxLayerCount - 1]; + }; + static_assert(util::is_pod::value); + + class HierarchicalIntegrityVerificationStorageControlArea { + NON_COPYABLE(HierarchicalIntegrityVerificationStorageControlArea); + NON_MOVEABLE(HierarchicalIntegrityVerificationStorageControlArea); + public: + static constexpr size_t HashSize = crypto::Sha256Generator::HashSize; + + struct InputParam { + size_t level_block_size[IntegrityMaxLayerCount - 1]; + }; + static_assert(util::is_pod::value); + private: + fs::SubStorage m_storage; + HierarchicalIntegrityVerificationMetaInformation m_meta; + public: + static Result QuerySize(HierarchicalIntegrityVerificationSizeSet *out, const InputParam &input_param, s32 layer_count, s64 data_size); + /* TODO Format */ + static Result Expand(fs::SubStorage meta_storage, const HierarchicalIntegrityVerificationMetaInformation &meta); + public: + HierarchicalIntegrityVerificationStorageControlArea() { /* ... */ } + + Result Initialize(fs::SubStorage meta_storage); + void Finalize(); + + u32 GetMasterHashSize() const { return m_meta.master_hash_size; } + void GetLevelHashInfo(HierarchicalIntegrityVerificationInformation *out) { + AMS_ASSERT(out != nullptr); + *out = m_meta.level_hash_info; + } + }; + + class HierarchicalIntegrityVerificationStorage : public ::ams::fs::IStorage { + NON_COPYABLE(HierarchicalIntegrityVerificationStorage); + NON_MOVEABLE(HierarchicalIntegrityVerificationStorage); + private: + friend struct HierarchicalIntegrityVerificationMetaInformation; + protected: + static constexpr s64 HashSize = crypto::Sha256Generator::HashSize; + static constexpr size_t MaxLayers = IntegrityMaxLayerCount; + public: + using GenerateRandomFunction = void (*)(void *dst, size_t size); + + class HierarchicalStorageInformation { + public: + enum { + MasterStorage = 0, + Layer1Storage = 1, + Layer2Storage = 2, + Layer3Storage = 3, + Layer4Storage = 4, + Layer5Storage = 5, + DataStorage = 6, + }; + private: + fs::SubStorage m_storages[DataStorage + 1]; + public: + void SetMasterHashStorage(fs::SubStorage s) { m_storages[MasterStorage] = s; } + void SetLayer1HashStorage(fs::SubStorage s) { m_storages[Layer1Storage] = s; } + void SetLayer2HashStorage(fs::SubStorage s) { m_storages[Layer2Storage] = s; } + void SetLayer3HashStorage(fs::SubStorage s) { m_storages[Layer3Storage] = s; } + void SetLayer4HashStorage(fs::SubStorage s) { m_storages[Layer4Storage] = s; } + void SetLayer5HashStorage(fs::SubStorage s) { m_storages[Layer5Storage] = s; } + void SetDataStorage(fs::SubStorage s) { m_storages[DataStorage] = s; } + + fs::SubStorage &operator[](s32 index) { + AMS_ASSERT(MasterStorage <= index && index <= DataStorage); + return m_storages[index]; + } + }; + private: + static GenerateRandomFunction s_generate_random; + + static void SetGenerateRandomFunction(GenerateRandomFunction func) { + s_generate_random = func; + } + private: + FileSystemBufferManagerSet *m_buffers; + os::SdkRecursiveMutex *m_mutex; + IntegrityVerificationStorage m_verify_storages[MaxLayers - 1]; + BlockCacheBufferedStorage m_buffer_storages[MaxLayers - 1]; + s64 m_data_size; + s32 m_max_layers; + bool m_is_written_for_rollback; + public: + HierarchicalIntegrityVerificationStorage() : m_buffers(nullptr), m_mutex(nullptr), m_data_size(-1), m_is_written_for_rollback(false) { /* ... */ } + virtual ~HierarchicalIntegrityVerificationStorage() override { this->Finalize(); } + + Result Initialize(const HierarchicalIntegrityVerificationInformation &info, HierarchicalStorageInformation storage, FileSystemBufferManagerSet *bufs, IHash256GeneratorFactory *hgf, os::SdkRecursiveMutex *mtx, fs::StorageType storage_type); + void Finalize(); + + virtual Result Read(s64 offset, void *buffer, size_t size) override; + virtual Result Write(s64 offset, const void *buffer, size_t size) override; + + virtual Result SetSize(s64 size) override { AMS_UNUSED(size); return fs::ResultUnsupportedOperationInHierarchicalIntegrityVerificationStorageA(); } + virtual Result GetSize(s64 *out) override; + + virtual Result Flush() override; + + virtual Result OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) override; + using IStorage::OperateRange; + + Result Commit(); + Result OnRollback(); + + bool IsInitialized() const { + return m_data_size >= 0; + } + + bool IsWrittenForRollback() const { + return m_is_written_for_rollback; + } + + FileSystemBufferManagerSet *GetBuffers() { + return m_buffers; + } + + void GetParameters(HierarchicalIntegrityVerificationStorageControlArea::InputParam *out) const { + AMS_ASSERT(out != nullptr); + for (auto level = 0; level <= m_max_layers - 2; ++level) { + out->level_block_size[level] = static_cast(m_verify_storages[level].GetBlockSize()); + } + } + + s64 GetL1HashVerificationBlockSize() const { + return m_verify_storages[m_max_layers - 2].GetBlockSize(); + } + + fs::SubStorage GetL1HashStorage() { + return fs::SubStorage(std::addressof(m_buffer_storages[m_max_layers - 3]), 0, util::DivideUp(m_data_size, this->GetL1HashVerificationBlockSize())); + } + }; + +} diff --git a/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_integrity_romfs_storage.hpp b/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_integrity_romfs_storage.hpp index cc888a4..15d1e5f 100644 --- a/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_integrity_romfs_storage.hpp +++ b/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_integrity_romfs_storage.hpp @@ -19,7 +19,7 @@ #include #include #include -#include +#include namespace ams::fssystem { @@ -28,8 +28,8 @@ class IntegrityRomFsStorage : public ::ams::fs::IStorage, public ::ams::fs::impl::Newable { private: - save::HierarchicalIntegrityVerificationStorage m_integrity_storage; - save::FileSystemBufferManagerSet m_buffers; + HierarchicalIntegrityVerificationStorage m_integrity_storage; + FileSystemBufferManagerSet m_buffers; os::SdkRecursiveMutex m_mutex; Hash m_master_hash; std::unique_ptr m_master_hash_storage; @@ -37,7 +37,7 @@ IntegrityRomFsStorage() : m_mutex() { /* ... */ } virtual ~IntegrityRomFsStorage() override { this->Finalize(); } - Result Initialize(save::HierarchicalIntegrityVerificationInformation level_hash_info, Hash master_hash, save::HierarchicalIntegrityVerificationStorage::HierarchicalStorageInformation storage_info, fs::IBufferManager *bm, IHash256GeneratorFactory *hgf); + Result Initialize(HierarchicalIntegrityVerificationInformation level_hash_info, Hash master_hash, HierarchicalIntegrityVerificationStorage::HierarchicalStorageInformation storage_info, fs::IBufferManager *bm, IHash256GeneratorFactory *hgf); void Finalize(); virtual Result Read(s64 offset, void *buffer, size_t size) override { @@ -66,7 +66,7 @@ return m_integrity_storage.Commit(); } - save::FileSystemBufferManagerSet *GetBuffers() { + FileSystemBufferManagerSet *GetBuffers() { return m_integrity_storage.GetBuffers(); } }; diff --git a/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_integrity_verification_storage.hpp b/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_integrity_verification_storage.hpp new file mode 100644 index 0000000..5e0aa32 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/fssystem/fssystem_integrity_verification_storage.hpp @@ -0,0 +1,98 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace ams::fssystem { + + class IntegrityVerificationStorage : public ::ams::fs::IStorage { + NON_COPYABLE(IntegrityVerificationStorage); + NON_MOVEABLE(IntegrityVerificationStorage); + public: + static constexpr s64 HashSize = crypto::Sha256Generator::HashSize; + + struct BlockHash { + u8 hash[HashSize]; + }; + static_assert(util::is_pod::value); + private: + fs::SubStorage m_hash_storage; + fs::SubStorage m_data_storage; + s64 m_verification_block_size; + s64 m_verification_block_order; + s64 m_upper_layer_verification_block_size; + s64 m_upper_layer_verification_block_order; + fs::IBufferManager *m_buffer_manager; + fs::HashSalt m_salt; + bool m_is_real_data; + fs::StorageType m_storage_type; + fssystem::IHash256GeneratorFactory *m_hash_generator_factory; + public: + IntegrityVerificationStorage() : m_verification_block_size(0), m_verification_block_order(0), m_upper_layer_verification_block_size(0), m_upper_layer_verification_block_order(0), m_buffer_manager(nullptr) { /* ... */ } + virtual ~IntegrityVerificationStorage() override { this->Finalize(); } + + Result Initialize(fs::SubStorage hs, fs::SubStorage ds, s64 verif_block_size, s64 upper_layer_verif_block_size, fs::IBufferManager *bm, fssystem::IHash256GeneratorFactory *hgf, const fs::HashSalt &salt, bool is_real_data, fs::StorageType storage_type); + void Finalize(); + + virtual Result Read(s64 offset, void *buffer, size_t size) override; + virtual Result Write(s64 offset, const void *buffer, size_t size) override; + + virtual Result SetSize(s64 size) override { AMS_UNUSED(size); return fs::ResultUnsupportedOperationInIntegrityVerificationStorageA(); } + virtual Result GetSize(s64 *out) override; + + virtual Result Flush() override; + + virtual Result OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) override; + using IStorage::OperateRange; + + void CalcBlockHash(BlockHash *out, const void *buffer, size_t block_size) const { + auto generator = m_hash_generator_factory->Create(); + return this->CalcBlockHash(out, buffer, block_size, generator); + } + + s64 GetBlockSize() const { + return m_verification_block_size; + } + private: + Result ReadBlockSignature(void *dst, size_t dst_size, s64 offset, size_t size); + Result WriteBlockSignature(const void *src, size_t src_size, s64 offset, size_t size); + Result VerifyHash(const void *buf, BlockHash *hash, std::unique_ptr &generator); + + void CalcBlockHash(BlockHash *out, const void *buffer, size_t block_size, std::unique_ptr &generator) const; + + void CalcBlockHash(BlockHash *out, const void *buffer, std::unique_ptr &generator) const { + return this->CalcBlockHash(out, buffer, static_cast(m_verification_block_size), generator); + } + + Result IsCleared(bool *is_cleared, const BlockHash &hash); + private: + static void SetValidationBit(BlockHash *hash) { + AMS_ASSERT(hash != nullptr); + hash->hash[HashSize - 1] |= 0x80; + } + + static bool IsValidationBit(const BlockHash *hash) { + AMS_ASSERT(hash != nullptr); + return (hash->hash[HashSize - 1] & 0x80) != 0; + } + }; + +} diff --git a/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_block_cache_buffered_storage.hpp b/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_block_cache_buffered_storage.hpp deleted file mode 100644 index fb875c2..0000000 --- a/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_block_cache_buffered_storage.hpp +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (c) Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ams::fssystem::save { - - constexpr inline size_t IntegrityMinLayerCount = 2; - constexpr inline size_t IntegrityMaxLayerCount = 7; - constexpr inline size_t IntegrityLayerCountSave = 5; - constexpr inline size_t IntegrityLayerCountSaveDataMeta = 4; - - struct FileSystemBufferManagerSet { - fs::IBufferManager *buffers[IntegrityMaxLayerCount]; - }; - static_assert(util::is_pod::value); - - class BlockCacheBufferedStorage : public ::ams::fs::IStorage { - NON_COPYABLE(BlockCacheBufferedStorage); - NON_MOVEABLE(BlockCacheBufferedStorage); - public: - static constexpr size_t DefaultMaxCacheEntryCount = 24; - private: - using MemoryRange = fs::IBufferManager::MemoryRange; - - struct AccessRange { - s64 offset; - size_t size; - - s64 GetEndOffset() const { - return this->offset + this->size; - } - - bool IsIncluded(s64 ofs) const { - return this->offset <= ofs && ofs < this->GetEndOffset(); - } - }; - static_assert(util::is_pod::value); - - struct CacheEntry { - AccessRange range; - bool is_valid; - bool is_write_back; - bool is_cached; - bool is_flushing; - u16 lru_counter; - fs::IBufferManager::CacheHandle handle; - uintptr_t memory_address; - size_t memory_size; - - void Invalidate() { - this->is_write_back = false; - this->is_flushing = false; - } - - bool IsAllocated() const { - return this->is_valid && (this->is_write_back ? this->memory_address != 0 : this->handle != 0); - } - - bool IsWriteBack() const { - return this->is_write_back; - } - }; - static_assert(util::is_pod::value); - - using BlockCacheManager = ::ams::fssystem::impl::BlockCacheManager; - using CacheIndex = BlockCacheManager::CacheIndex; - - enum Flag : s32 { - Flag_KeepBurstMode = (1 << 8), - Flag_RealData = (1 << 10), - }; - private: - os::SdkRecursiveMutex *m_mutex; - IStorage *m_data_storage; - Result m_last_result; - s64 m_data_size; - size_t m_verification_block_size; - size_t m_verification_block_shift; - s32 m_flags; - s32 m_buffer_level; - fs::StorageType m_storage_type; - BlockCacheManager m_block_cache_manager; - public: - BlockCacheBufferedStorage(); - virtual ~BlockCacheBufferedStorage() override; - - Result Initialize(fs::IBufferManager *bm, os::SdkRecursiveMutex *mtx, IStorage *data, s64 data_size, size_t verif_block_size, s32 max_cache_entries, bool is_real_data, s8 buffer_level, bool is_keep_burst_mode, fs::StorageType storage_type); - void Finalize(); - - virtual Result Read(s64 offset, void *buffer, size_t size) override; - virtual Result Write(s64 offset, const void *buffer, size_t size) override; - - virtual Result SetSize(s64) override { R_THROW(fs::ResultUnsupportedOperationInBlockCacheBufferedStorageA()); } - virtual Result GetSize(s64 *out) override; - - virtual Result Flush() override; - - virtual Result OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) override; - using IStorage::OperateRange; - - Result Commit(); - Result OnRollback(); - - bool IsEnabledKeepBurstMode() const { - return (m_flags & Flag_KeepBurstMode) != 0; - } - - bool IsRealDataCache() const { - return (m_flags & Flag_RealData) != 0; - } - - void SetKeepBurstMode(bool en) { - if (en) { - m_flags |= Flag_KeepBurstMode; - } else { - m_flags &= ~Flag_KeepBurstMode; - } - } - - void SetRealDataCache(bool en) { - if (en) { - m_flags |= Flag_RealData; - } else { - m_flags &= ~Flag_RealData; - } - } - private: - Result FillZeroImpl(s64 offset, s64 size); - Result DestroySignatureImpl(s64 offset, s64 size); - Result InvalidateImpl(); - Result QueryRangeImpl(void *dst, size_t dst_size, s64 offset, s64 size); - - Result GetAssociateBuffer(MemoryRange *out_range, CacheEntry *out_entry, s64 offset, size_t ideal_size, bool is_allocate_for_write); - - Result StoreOrDestroyBuffer(CacheIndex *out, const MemoryRange &range, CacheEntry *entry); - - Result StoreOrDestroyBuffer(const MemoryRange &range, CacheEntry *entry) { - AMS_ASSERT(entry != nullptr); - - CacheIndex dummy; - R_RETURN(this->StoreOrDestroyBuffer(std::addressof(dummy), range, entry)); - } - - Result FlushCacheEntry(CacheIndex index, bool invalidate); - Result FlushRangeCacheEntries(s64 offset, s64 size, bool invalidate); - - Result FlushAllCacheEntries(); - Result InvalidateAllCacheEntries(); - Result ControlDirtiness(); - - Result UpdateLastResult(Result result); - - Result ReadHeadCache(MemoryRange *out_range, CacheEntry *out_entry, bool *out_cache_needed, s64 *offset, s64 *aligned_offset, s64 aligned_offset_end, char **buffer, size_t *size); - Result ReadTailCache(MemoryRange *out_range, CacheEntry *out_entry, bool *out_cache_needed, s64 offset, s64 aligned_offset, s64 *aligned_offset_end, char *buffer, size_t *size); - - Result BulkRead(s64 offset, void *buffer, size_t size, MemoryRange *range_head, MemoryRange *range_tail, CacheEntry *entry_head, CacheEntry *entry_tail, bool head_cache_needed, bool tail_cache_needed); - }; - -} diff --git a/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_buffered_storage.hpp b/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_buffered_storage.hpp deleted file mode 100644 index 9fdaec9..0000000 --- a/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_buffered_storage.hpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#pragma once -#include -#include -#include -#include -#include - -namespace ams::fssystem::save { - - class BufferedStorage : public ::ams::fs::IStorage { - NON_COPYABLE(BufferedStorage); - NON_MOVEABLE(BufferedStorage); - private: - class Cache; - class UniqueCache; - class SharedCache; - private: - fs::SubStorage m_base_storage; - fs::IBufferManager *m_buffer_manager; - size_t m_block_size; - s64 m_base_storage_size; - std::unique_ptr m_caches; - s32 m_cache_count; - Cache *m_next_acquire_cache; - Cache *m_next_fetch_cache; - os::SdkMutex m_mutex; - bool m_bulk_read_enabled; - public: - BufferedStorage(); - virtual ~BufferedStorage(); - - Result Initialize(fs::SubStorage base_storage, fs::IBufferManager *buffer_manager, size_t block_size, s32 buffer_count); - void Finalize(); - - bool IsInitialized() const { return m_caches != nullptr; } - - virtual Result Read(s64 offset, void *buffer, size_t size) override; - virtual Result Write(s64 offset, const void *buffer, size_t size) override; - - virtual Result GetSize(s64 *out) override; - virtual Result SetSize(s64 size) override; - - virtual Result Flush() override; - virtual Result OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) override; - using IStorage::OperateRange; - - void InvalidateCaches(); - - fs::IBufferManager *GetBufferManager() const { return m_buffer_manager; } - - void EnableBulkRead() { m_bulk_read_enabled = true; } - private: - Result PrepareAllocation(); - Result ControlDirtiness(); - Result ReadCore(s64 offset, void *buffer, size_t size); - - bool ReadHeadCache(s64 *offset, void *buffer, size_t *size, s64 *buffer_offset); - bool ReadTailCache(s64 offset, void *buffer, size_t *size, s64 buffer_offset); - - Result BulkRead(s64 offset, void *buffer, size_t size, bool head_cache_needed, bool tail_cache_needed); - - Result WriteCore(s64 offset, const void *buffer, size_t size); - }; - -} diff --git a/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_hierarchical_integrity_verification_storage.hpp b/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_hierarchical_integrity_verification_storage.hpp deleted file mode 100644 index 1ba245e..0000000 --- a/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_hierarchical_integrity_verification_storage.hpp +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright (c) Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ams::fssystem::save { - - struct HierarchicalIntegrityVerificationLevelInformation { - fs::Int64 offset; - fs::Int64 size; - s32 block_order; - u8 reserved[4]; - }; - static_assert(util::is_pod::value); - static_assert(sizeof(HierarchicalIntegrityVerificationLevelInformation) == 0x18); - static_assert(alignof(HierarchicalIntegrityVerificationLevelInformation) == 0x4); - - struct HierarchicalIntegrityVerificationInformation { - u32 max_layers; - HierarchicalIntegrityVerificationLevelInformation info[IntegrityMaxLayerCount - 1]; - fs::HashSalt seed; - - s64 GetLayeredHashSize() const { - return this->info[this->max_layers - 2].offset; - } - - s64 GetDataOffset() const { - return this->info[this->max_layers - 2].offset; - } - - s64 GetDataSize() const { - return this->info[this->max_layers - 2].size; - } - }; - static_assert(util::is_pod::value); - - struct HierarchicalIntegrityVerificationMetaInformation { - u32 magic; - u32 version; - u32 master_hash_size; - HierarchicalIntegrityVerificationInformation level_hash_info; - - /* TODO: Format */ - }; - static_assert(util::is_pod::value); - - struct HierarchicalIntegrityVerificationSizeSet { - s64 control_size; - s64 master_hash_size; - s64 layered_hash_sizes[IntegrityMaxLayerCount - 1]; - }; - static_assert(util::is_pod::value); - - class HierarchicalIntegrityVerificationStorageControlArea { - NON_COPYABLE(HierarchicalIntegrityVerificationStorageControlArea); - NON_MOVEABLE(HierarchicalIntegrityVerificationStorageControlArea); - public: - static constexpr size_t HashSize = crypto::Sha256Generator::HashSize; - - struct InputParam { - size_t level_block_size[IntegrityMaxLayerCount - 1]; - }; - static_assert(util::is_pod::value); - private: - fs::SubStorage m_storage; - HierarchicalIntegrityVerificationMetaInformation m_meta; - public: - static Result QuerySize(HierarchicalIntegrityVerificationSizeSet *out, const InputParam &input_param, s32 layer_count, s64 data_size); - /* TODO Format */ - static Result Expand(fs::SubStorage meta_storage, const HierarchicalIntegrityVerificationMetaInformation &meta); - public: - HierarchicalIntegrityVerificationStorageControlArea() { /* ... */ } - - Result Initialize(fs::SubStorage meta_storage); - void Finalize(); - - u32 GetMasterHashSize() const { return m_meta.master_hash_size; } - void GetLevelHashInfo(HierarchicalIntegrityVerificationInformation *out) { - AMS_ASSERT(out != nullptr); - *out = m_meta.level_hash_info; - } - }; - - class HierarchicalIntegrityVerificationStorage : public ::ams::fs::IStorage { - NON_COPYABLE(HierarchicalIntegrityVerificationStorage); - NON_MOVEABLE(HierarchicalIntegrityVerificationStorage); - private: - friend struct HierarchicalIntegrityVerificationMetaInformation; - protected: - static constexpr s64 HashSize = crypto::Sha256Generator::HashSize; - static constexpr size_t MaxLayers = IntegrityMaxLayerCount; - public: - using GenerateRandomFunction = void (*)(void *dst, size_t size); - - class HierarchicalStorageInformation { - public: - enum { - MasterStorage = 0, - Layer1Storage = 1, - Layer2Storage = 2, - Layer3Storage = 3, - Layer4Storage = 4, - Layer5Storage = 5, - DataStorage = 6, - }; - private: - fs::SubStorage m_storages[DataStorage + 1]; - public: - void SetMasterHashStorage(fs::SubStorage s) { m_storages[MasterStorage] = s; } - void SetLayer1HashStorage(fs::SubStorage s) { m_storages[Layer1Storage] = s; } - void SetLayer2HashStorage(fs::SubStorage s) { m_storages[Layer2Storage] = s; } - void SetLayer3HashStorage(fs::SubStorage s) { m_storages[Layer3Storage] = s; } - void SetLayer4HashStorage(fs::SubStorage s) { m_storages[Layer4Storage] = s; } - void SetLayer5HashStorage(fs::SubStorage s) { m_storages[Layer5Storage] = s; } - void SetDataStorage(fs::SubStorage s) { m_storages[DataStorage] = s; } - - fs::SubStorage &operator[](s32 index) { - AMS_ASSERT(MasterStorage <= index && index <= DataStorage); - return m_storages[index]; - } - }; - private: - static GenerateRandomFunction s_generate_random; - - static void SetGenerateRandomFunction(GenerateRandomFunction func) { - s_generate_random = func; - } - private: - FileSystemBufferManagerSet *m_buffers; - os::SdkRecursiveMutex *m_mutex; - IntegrityVerificationStorage m_verify_storages[MaxLayers - 1]; - BlockCacheBufferedStorage m_buffer_storages[MaxLayers - 1]; - s64 m_data_size; - s32 m_max_layers; - bool m_is_written_for_rollback; - public: - HierarchicalIntegrityVerificationStorage() : m_buffers(nullptr), m_mutex(nullptr), m_data_size(-1), m_is_written_for_rollback(false) { /* ... */ } - virtual ~HierarchicalIntegrityVerificationStorage() override { this->Finalize(); } - - Result Initialize(const HierarchicalIntegrityVerificationInformation &info, HierarchicalStorageInformation storage, FileSystemBufferManagerSet *bufs, IHash256GeneratorFactory *hgf, os::SdkRecursiveMutex *mtx, fs::StorageType storage_type); - void Finalize(); - - virtual Result Read(s64 offset, void *buffer, size_t size) override; - virtual Result Write(s64 offset, const void *buffer, size_t size) override; - - virtual Result SetSize(s64 size) override { AMS_UNUSED(size); return fs::ResultUnsupportedOperationInHierarchicalIntegrityVerificationStorageA(); } - virtual Result GetSize(s64 *out) override; - - virtual Result Flush() override; - - virtual Result OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) override; - using IStorage::OperateRange; - - Result Commit(); - Result OnRollback(); - - bool IsInitialized() const { - return m_data_size >= 0; - } - - bool IsWrittenForRollback() const { - return m_is_written_for_rollback; - } - - FileSystemBufferManagerSet *GetBuffers() { - return m_buffers; - } - - void GetParameters(HierarchicalIntegrityVerificationStorageControlArea::InputParam *out) const { - AMS_ASSERT(out != nullptr); - for (auto level = 0; level <= m_max_layers - 2; ++level) { - out->level_block_size[level] = static_cast(m_verify_storages[level].GetBlockSize()); - } - } - - s64 GetL1HashVerificationBlockSize() const { - return m_verify_storages[m_max_layers - 2].GetBlockSize(); - } - - fs::SubStorage GetL1HashStorage() { - return fs::SubStorage(std::addressof(m_buffer_storages[m_max_layers - 3]), 0, util::DivideUp(m_data_size, this->GetL1HashVerificationBlockSize())); - } - }; - -} diff --git a/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_integrity_verification_storage.hpp b/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_integrity_verification_storage.hpp deleted file mode 100644 index b162af5..0000000 --- a/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_integrity_verification_storage.hpp +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ams::fssystem::save { - - class IntegrityVerificationStorage : public ::ams::fs::IStorage { - NON_COPYABLE(IntegrityVerificationStorage); - NON_MOVEABLE(IntegrityVerificationStorage); - public: - static constexpr s64 HashSize = crypto::Sha256Generator::HashSize; - - struct BlockHash { - u8 hash[HashSize]; - }; - static_assert(util::is_pod::value); - private: - fs::SubStorage m_hash_storage; - fs::SubStorage m_data_storage; - s64 m_verification_block_size; - s64 m_verification_block_order; - s64 m_upper_layer_verification_block_size; - s64 m_upper_layer_verification_block_order; - fs::IBufferManager *m_buffer_manager; - fs::HashSalt m_salt; - bool m_is_real_data; - fs::StorageType m_storage_type; - fssystem::IHash256GeneratorFactory *m_hash_generator_factory; - public: - IntegrityVerificationStorage() : m_verification_block_size(0), m_verification_block_order(0), m_upper_layer_verification_block_size(0), m_upper_layer_verification_block_order(0), m_buffer_manager(nullptr) { /* ... */ } - virtual ~IntegrityVerificationStorage() override { this->Finalize(); } - - Result Initialize(fs::SubStorage hs, fs::SubStorage ds, s64 verif_block_size, s64 upper_layer_verif_block_size, fs::IBufferManager *bm, fssystem::IHash256GeneratorFactory *hgf, const fs::HashSalt &salt, bool is_real_data, fs::StorageType storage_type); - void Finalize(); - - virtual Result Read(s64 offset, void *buffer, size_t size) override; - virtual Result Write(s64 offset, const void *buffer, size_t size) override; - - virtual Result SetSize(s64 size) override { AMS_UNUSED(size); return fs::ResultUnsupportedOperationInIntegrityVerificationStorageA(); } - virtual Result GetSize(s64 *out) override; - - virtual Result Flush() override; - - virtual Result OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) override; - using IStorage::OperateRange; - - void CalcBlockHash(BlockHash *out, const void *buffer, size_t block_size) const { - auto generator = m_hash_generator_factory->Create(); - return this->CalcBlockHash(out, buffer, block_size, generator); - } - - s64 GetBlockSize() const { - return m_verification_block_size; - } - private: - Result ReadBlockSignature(void *dst, size_t dst_size, s64 offset, size_t size); - Result WriteBlockSignature(const void *src, size_t src_size, s64 offset, size_t size); - Result VerifyHash(const void *buf, BlockHash *hash, std::unique_ptr &generator); - - void CalcBlockHash(BlockHash *out, const void *buffer, size_t block_size, std::unique_ptr &generator) const; - - void CalcBlockHash(BlockHash *out, const void *buffer, std::unique_ptr &generator) const { - return this->CalcBlockHash(out, buffer, static_cast(m_verification_block_size), generator); - } - - Result IsCleared(bool *is_cleared, const BlockHash &hash); - private: - static void SetValidationBit(BlockHash *hash) { - AMS_ASSERT(hash != nullptr); - hash->hash[HashSize - 1] |= 0x80; - } - - static bool IsValidationBit(const BlockHash *hash) { - AMS_ASSERT(hash != nullptr); - return (hash->hash[HashSize - 1] & 0x80) != 0; - } - }; - -} diff --git a/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_save_types.hpp b/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_save_types.hpp deleted file mode 100644 index 3522552..0000000 --- a/libraries/libstratosphere/include/stratosphere/fssystem/save/fssystem_save_types.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#pragma once -#include -#include -#include -#include -#include - -namespace ams::fssystem::save { - - constexpr inline bool IsPowerOfTwo(s32 val) { - return util::IsPowerOfTwo(val); - } - - constexpr inline u32 ILog2(u32 val) { - AMS_ASSERT(val > 0); - return (BITSIZEOF(u32) - 1 - dbm::CountLeadingZeros(val)); - } - - constexpr inline u32 CeilPowerOfTwo(u32 val) { - if (val == 0) { - return 1; - } - return ((1u << (BITSIZEOF(u32) - 1)) >> (dbm::CountLeadingZeros(val - 1) - 1)); - } - -} diff --git a/libraries/libstratosphere/source/fssystem/fssystem_block_cache_buffered_storage.cpp b/libraries/libstratosphere/source/fssystem/fssystem_block_cache_buffered_storage.cpp new file mode 100644 index 0000000..f2d6d2b --- /dev/null +++ b/libraries/libstratosphere/source/fssystem/fssystem_block_cache_buffered_storage.cpp @@ -0,0 +1,1072 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::fssystem { + + BlockCacheBufferedStorage::BlockCacheBufferedStorage() : m_mutex(), m_data_storage(), m_last_result(ResultSuccess()), m_data_size(), m_verification_block_size(), m_verification_block_shift(), m_flags(), m_buffer_level(-1), m_block_cache_manager() + { + /* ... */ + } + + BlockCacheBufferedStorage::~BlockCacheBufferedStorage() { + this->Finalize(); + } + + Result BlockCacheBufferedStorage::Initialize(fs::IBufferManager *bm, os::SdkRecursiveMutex *mtx, IStorage *data, s64 data_size, size_t verif_block_size, s32 max_cache_entries, bool is_real_data, s8 buffer_level, bool is_keep_burst_mode, fs::StorageType storage_type) { + /* Validate preconditions. */ + AMS_ASSERT(data != nullptr); + AMS_ASSERT(bm != nullptr); + AMS_ASSERT(mtx != nullptr); + AMS_ASSERT(m_mutex == nullptr); + AMS_ASSERT(m_data_storage == nullptr); + AMS_ASSERT(max_cache_entries > 0); + + /* Initialize our manager. */ + R_TRY(m_block_cache_manager.Initialize(bm, max_cache_entries)); + + /* Set members. */ + m_mutex = mtx; + m_data_storage = data; + m_data_size = data_size; + m_verification_block_size = verif_block_size; + m_last_result = ResultSuccess(); + m_flags = 0; + m_buffer_level = buffer_level; + m_storage_type = storage_type; + + /* Calculate block shift. */ + m_verification_block_shift = ILog2(static_cast(verif_block_size)); + AMS_ASSERT(static_cast(UINT64_C(1) << m_verification_block_shift) == m_verification_block_size); + + /* Set burst mode. */ + this->SetKeepBurstMode(is_keep_burst_mode); + + /* Set real data cache. */ + this->SetRealDataCache(is_real_data); + + R_SUCCEED(); + } + + void BlockCacheBufferedStorage::Finalize() { + if (m_block_cache_manager.IsInitialized()) { + /* Invalidate all cache entries. */ + this->InvalidateAllCacheEntries(); + + /* Finalize our block cache manager. */ + m_block_cache_manager.Finalize(); + + /* Clear members. */ + m_mutex = nullptr; + m_data_storage = nullptr; + m_data_size = 0; + m_verification_block_size = 0; + m_verification_block_shift = 0; + } + } + + Result BlockCacheBufferedStorage::Read(s64 offset, void *buffer, size_t size) { + /* Validate pre-conditions. */ + AMS_ASSERT(m_data_storage != nullptr); + AMS_ASSERT(m_block_cache_manager.IsInitialized()); + + /* Ensure we aren't already in a failed state. */ + R_TRY(m_last_result); + + /* Succeed if zero-size. */ + R_SUCCEED_IF(size == 0); + + /* Validate arguments. */ + R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); + + /* Determine the extents to read. */ + s64 read_offset = offset; + size_t read_size = size; + + R_UNLESS(read_offset < m_data_size, fs::ResultInvalidOffset()); + + if (static_cast(read_offset + read_size) > m_data_size) { + read_size = static_cast(m_data_size - read_offset); + } + + /* Determine the aligned range to read. */ + const size_t block_alignment = m_verification_block_size; + s64 aligned_offset = util::AlignDown(read_offset, block_alignment); + s64 aligned_offset_end = util::AlignUp(read_offset + read_size, block_alignment); + + AMS_ASSERT(0 <= aligned_offset && aligned_offset_end <= static_cast(util::AlignUp(m_data_size, block_alignment))); + + /* Try to read using cache. */ + char *dst = static_cast(buffer); + { + /* Determine if we can do bulk reads. */ + constexpr s64 BulkReadSizeMax = 2_MB; + const bool bulk_read_enabled = (read_offset != aligned_offset || static_cast(read_offset + read_size) != aligned_offset_end) && aligned_offset_end - aligned_offset <= BulkReadSizeMax; + + /* Read the head cache. */ + CacheEntry head_entry = {}; + MemoryRange head_range = {}; + bool head_cache_needed = true; + R_TRY(this->ReadHeadCache(std::addressof(head_range), std::addressof(head_entry), std::addressof(head_cache_needed), std::addressof(read_offset), std::addressof(aligned_offset), aligned_offset_end, std::addressof(dst), std::addressof(read_size))); + + /* We may be done after reading the head cache, so check if we are. */ + R_SUCCEED_IF(aligned_offset >= aligned_offset_end); + + /* Ensure we destroy the head buffer. */ + auto head_guard = SCOPE_GUARD { m_block_cache_manager.ReleaseCacheEntry(std::addressof(head_entry), head_range); }; + + /* Read the tail cache. */ + CacheEntry tail_entry = {}; + MemoryRange tail_range = {}; + bool tail_cache_needed = true; + R_TRY(this->ReadTailCache(std::addressof(tail_range), std::addressof(tail_entry), std::addressof(tail_cache_needed), read_offset, aligned_offset, std::addressof(aligned_offset_end), dst, std::addressof(read_size))); + + /* We may be done after reading the tail cache, so check if we are. */ + R_SUCCEED_IF(aligned_offset >= aligned_offset_end); + + /* Ensure that we destroy the tail buffer. */ + auto tail_guard = SCOPE_GUARD { m_block_cache_manager.ReleaseCacheEntry(std::addressof(tail_entry), tail_range); }; + + /* Try to do a bulk read. */ + if (bulk_read_enabled) { + /* The bulk read will destroy our head/tail buffers. */ + head_guard.Cancel(); + tail_guard.Cancel(); + + do { + /* Do the bulk read. If we fail due to pooled buffer allocation failing, fall back to the normal read codepath. */ + R_TRY_CATCH(this->BulkRead(read_offset, dst, read_size, std::addressof(head_range), std::addressof(tail_range), std::addressof(head_entry), std::addressof(tail_entry), head_cache_needed, tail_cache_needed)) { + R_CATCH(fs::ResultAllocationFailurePooledBufferNotEnoughSize) { break; } + } R_END_TRY_CATCH; + + /* Se successfully did a bulk read, so we're done. */ + R_SUCCEED(); + } while (0); + } + } + + /* Read the data using non-bulk reads. */ + while (aligned_offset < aligned_offset_end) { + /* Ensure that there is data for us to read. */ + AMS_ASSERT(read_size > 0); + + /* If conditions allow us to, read in burst mode. This doesn't use the cache. */ + if (this->IsEnabledKeepBurstMode() && read_offset == aligned_offset && (block_alignment * 2 <= read_size)) { + const size_t aligned_size = util::AlignDown(read_size, block_alignment); + + /* Flush the entries. */ + R_TRY(this->UpdateLastResult(this->FlushRangeCacheEntries(read_offset, aligned_size, false))); + + /* Read the data. */ + R_TRY(this->UpdateLastResult(m_data_storage->Read(read_offset, dst, aligned_size))); + + /* Advance. */ + dst += aligned_size; + read_offset += aligned_size; + read_size -= aligned_size; + aligned_offset += aligned_size; + } else { + /* Get the buffer associated with what we're reading. */ + CacheEntry entry; + MemoryRange range; + R_TRY(this->UpdateLastResult(this->GetAssociateBuffer(std::addressof(range), std::addressof(entry), aligned_offset, static_cast(aligned_offset_end - aligned_offset), true))); + + /* Determine where to read data into, and ensure that our entry is aligned. */ + char *src = reinterpret_cast(range.first); + AMS_ASSERT(util::IsAligned(entry.range.size, block_alignment)); + + /* If the entry isn't cached, read the data. */ + if (!entry.is_cached) { + if (const Result result = m_data_storage->Read(entry.range.offset, src, entry.range.size); R_FAILED(result)) { + m_block_cache_manager.ReleaseCacheEntry(std::addressof(entry), range); + return this->UpdateLastResult(result); + } + entry.is_cached = true; + } + + /* Validate the entry extents. */ + AMS_ASSERT(static_cast(entry.range.offset) <= aligned_offset); + AMS_ASSERT(aligned_offset < entry.range.GetEndOffset()); + AMS_ASSERT(aligned_offset <= read_offset); + + /* Copy the data. */ + { + /* Determine where and how much to copy. */ + const s64 buffer_offset = read_offset - entry.range.offset; + const size_t copy_size = std::min(read_size, static_cast(entry.range.GetEndOffset() - read_offset)); + + /* Actually copy the data. */ + std::memcpy(dst, src + buffer_offset, copy_size); + + /* Advance. */ + dst += copy_size; + read_offset += copy_size; + read_size -= copy_size; + } + + /* Release the cache entry. */ + R_TRY(this->UpdateLastResult(this->StoreOrDestroyBuffer(range, std::addressof(entry)))); + aligned_offset = entry.range.GetEndOffset(); + } + } + + /* Ensure that we read all the data. */ + AMS_ASSERT(read_size == 0); + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::Write(s64 offset, const void *buffer, size_t size) { + /* Validate pre-conditions. */ + AMS_ASSERT(m_data_storage != nullptr); + AMS_ASSERT(m_block_cache_manager.IsInitialized()); + + /* Ensure we aren't already in a failed state. */ + R_TRY(m_last_result); + + /* Succeed if zero-size. */ + R_SUCCEED_IF(size == 0); + + /* Validate arguments. */ + R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); + + /* Determine the extents to read. */ + R_UNLESS(offset < m_data_size, fs::ResultInvalidOffset()); + + if (static_cast(offset + size) > m_data_size) { + size = static_cast(m_data_size - offset); + } + + /* The actual extents may be zero-size, so succeed if that's the case. */ + R_SUCCEED_IF(size == 0); + + /* Determine the aligned range to read. */ + const size_t block_alignment = m_verification_block_size; + s64 aligned_offset = util::AlignDown(offset, block_alignment); + const s64 aligned_offset_end = util::AlignUp(offset + size, block_alignment); + + AMS_ASSERT(0 <= aligned_offset && aligned_offset_end <= static_cast(util::AlignUp(m_data_size, block_alignment))); + + /* Write the data. */ + const u8 *src = static_cast(buffer); + while (aligned_offset < aligned_offset_end) { + /* If conditions allow us to, write in burst mode. This doesn't use the cache. */ + if (this->IsEnabledKeepBurstMode() && offset == aligned_offset && (block_alignment * 2 <= size)) { + const size_t aligned_size = util::AlignDown(size, block_alignment); + + /* Flush the entries. */ + R_TRY(this->UpdateLastResult(this->FlushRangeCacheEntries(offset, aligned_size, true))); + + /* Read the data. */ + R_TRY(this->UpdateLastResult(m_data_storage->Write(offset, src, aligned_size))); + + /* Set blocking buffer manager allocations. */ + buffers::EnableBlockingBufferManagerAllocation(); + + /* Advance. */ + src += aligned_size; + offset += aligned_size; + size -= aligned_size; + aligned_offset += aligned_size; + } else { + /* Get the buffer associated with what we're writing. */ + CacheEntry entry; + MemoryRange range; + R_TRY(this->UpdateLastResult(this->GetAssociateBuffer(std::addressof(range), std::addressof(entry), aligned_offset, static_cast(aligned_offset_end - aligned_offset), true))); + + /* Determine where to write data into. */ + char *dst = reinterpret_cast(range.first); + + /* If the entry isn't cached and we're writing a partial entry, read in the entry. */ + if (!entry.is_cached && ((offset != entry.range.offset) || (offset + size < static_cast(entry.range.GetEndOffset())))) { + if (Result result = m_data_storage->Read(entry.range.offset, dst, entry.range.size); R_FAILED(result)) { + m_block_cache_manager.ReleaseCacheEntry(std::addressof(entry), range); + return this->UpdateLastResult(result); + } + } + entry.is_cached = true; + + /* Validate the entry extents. */ + AMS_ASSERT(static_cast(entry.range.offset) <= aligned_offset); + AMS_ASSERT(aligned_offset < entry.range.GetEndOffset()); + AMS_ASSERT(aligned_offset <= offset); + + /* Copy the data. */ + { + /* Determine where and how much to copy. */ + const s64 buffer_offset = offset - entry.range.offset; + const size_t copy_size = std::min(size, static_cast(entry.range.GetEndOffset() - offset)); + + /* Actually copy the data. */ + std::memcpy(dst + buffer_offset, src, copy_size); + + /* Advance. */ + src += copy_size; + offset += copy_size; + size -= copy_size; + } + + /* Set the entry as write-back. */ + entry.is_write_back = true; + + /* Set blocking buffer manager allocations. */ + buffers::EnableBlockingBufferManagerAllocation(); + + /* Store the associated buffer. */ + CacheIndex index; + R_TRY(this->UpdateLastResult(this->StoreOrDestroyBuffer(std::addressof(index), range, std::addressof(entry)))); + + /* Set the after aligned offset. */ + aligned_offset = entry.range.GetEndOffset(); + + /* If we need to, flush the cache entry. */ + if (index >= 0 && IsEnabledKeepBurstMode() && offset == aligned_offset && (block_alignment * 2 <= size)) { + R_TRY(this->UpdateLastResult(this->FlushCacheEntry(index, false))); + } + + } + } + + /* Ensure that didn't end up in a failure state. */ + R_TRY(m_last_result); + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::GetSize(s64 *out) { + /* Validate pre-conditions. */ + AMS_ASSERT(out != nullptr); + AMS_ASSERT(m_data_storage != nullptr); + + /* Set the size. */ + *out = m_data_size; + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::Flush() { + /* Validate pre-conditions. */ + AMS_ASSERT(m_data_storage != nullptr); + AMS_ASSERT(m_block_cache_manager.IsInitialized()); + + /* Ensure we aren't already in a failed state. */ + R_TRY(m_last_result); + + /* Flush all cache entries. */ + R_TRY(this->UpdateLastResult(this->FlushAllCacheEntries())); + + /* Flush the data storage. */ + R_TRY(this->UpdateLastResult(m_data_storage->Flush())); + + /* Set blocking buffer manager allocations. */ + buffers::EnableBlockingBufferManagerAllocation(); + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) { + AMS_UNUSED(src, src_size); + + /* Validate pre-conditions. */ + AMS_ASSERT(m_data_storage != nullptr); + + switch (op_id) { + case fs::OperationId::FillZero: + { + R_TRY(this->FillZeroImpl(offset, size)); + R_SUCCEED(); + } + case fs::OperationId::DestroySignature: + { + R_TRY(this->DestroySignatureImpl(offset, size)); + R_SUCCEED(); + } + case fs::OperationId::Invalidate: + { + R_UNLESS(m_storage_type != fs::StorageType_SaveData, fs::ResultUnsupportedOperationInBlockCacheBufferedStorageB()); + R_TRY(this->InvalidateImpl()); + R_SUCCEED(); + } + case fs::OperationId::QueryRange: + { + R_TRY(this->QueryRangeImpl(dst, dst_size, offset, size)); + R_SUCCEED(); + } + default: + R_THROW(fs::ResultUnsupportedOperationInBlockCacheBufferedStorageC()); + } + } + + Result BlockCacheBufferedStorage::Commit() { + /* Validate pre-conditions. */ + AMS_ASSERT(m_data_storage != nullptr); + AMS_ASSERT(m_block_cache_manager.IsInitialized()); + + /* Ensure we aren't already in a failed state. */ + R_TRY(m_last_result); + + /* Flush all cache entries. */ + R_TRY(this->UpdateLastResult(this->FlushAllCacheEntries())); + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::OnRollback() { + /* Validate pre-conditions. */ + AMS_ASSERT(m_block_cache_manager.IsInitialized()); + + /* Ensure we aren't already in a failed state. */ + R_TRY(m_last_result); + + /* Release all valid entries back to the buffer manager. */ + const auto max_cache_entry_count = m_block_cache_manager.GetCount(); + for (auto index = 0; index < max_cache_entry_count; index++) { + if (const auto &entry = m_block_cache_manager[index]; entry.is_valid) { + m_block_cache_manager.InvalidateCacheEntry(index); + } + } + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::FillZeroImpl(s64 offset, s64 size) { + /* Ensure we aren't already in a failed state. */ + R_TRY(m_last_result); + + /* Get our storage size. */ + s64 storage_size = 0; + R_TRY(this->UpdateLastResult(m_data_storage->GetSize(std::addressof(storage_size)))); + + /* Check the access range. */ + R_UNLESS(0 <= offset && offset < storage_size, fs::ResultInvalidOffset()); + + /* Determine the extents to data signature for. */ + auto start_offset = util::AlignDown(offset, m_verification_block_size); + auto end_offset = util::AlignUp(std::min(offset + size, storage_size), m_verification_block_size); + + /* Flush the entries. */ + R_TRY(this->UpdateLastResult(this->FlushRangeCacheEntries(offset, size, true))); + + /* Handle any data before or after the aligned range. */ + if (start_offset < offset || offset + size < end_offset) { + /* Allocate a work buffer. */ + std::unique_ptr work = fs::impl::MakeUnique(m_verification_block_size); + R_UNLESS(work != nullptr, fs::ResultAllocationFailureInBlockCacheBufferedStorageB()); + + /* Handle data before the aligned range. */ + if (start_offset < offset) { + /* Read the block. */ + R_TRY(this->UpdateLastResult(m_data_storage->Read(start_offset, work.get(), m_verification_block_size))); + + /* Determine the partial extents to clear. */ + const auto clear_offset = static_cast(offset - start_offset); + const auto clear_size = static_cast(std::min(static_cast(m_verification_block_size - clear_offset), size)); + + /* Clear the partial block. */ + std::memset(work.get() + clear_offset, 0, clear_size); + + /* Write the partially cleared block. */ + R_TRY(this->UpdateLastResult(m_data_storage->Write(start_offset, work.get(), m_verification_block_size))); + + /* Update the start offset. */ + start_offset += m_verification_block_size; + + /* Set blocking buffer manager allocations. */ + buffers::EnableBlockingBufferManagerAllocation(); + } + + /* Handle data after the aligned range. */ + if (start_offset < offset + size && offset + size < end_offset) { + /* Read the block. */ + const auto last_offset = end_offset - m_verification_block_size; + R_TRY(this->UpdateLastResult(m_data_storage->Read(last_offset, work.get(), m_verification_block_size))); + + /* Clear the partial block. */ + const auto clear_size = static_cast((offset + size) - last_offset); + std::memset(work.get(), 0, clear_size); + + /* Write the partially cleared block. */ + R_TRY(this->UpdateLastResult(m_data_storage->Write(last_offset, work.get(), m_verification_block_size))); + + /* Update the end offset. */ + end_offset -= m_verification_block_size; + + /* Set blocking buffer manager allocations. */ + buffers::EnableBlockingBufferManagerAllocation(); + } + } + + /* We're done if there's no data to clear. */ + R_SUCCEED_IF(start_offset == end_offset); + + /* Clear the signature for the aligned range. */ + R_TRY(this->UpdateLastResult(m_data_storage->OperateRange(fs::OperationId::FillZero, start_offset, end_offset - start_offset))); + + /* Set blocking buffer manager allocations. */ + buffers::EnableBlockingBufferManagerAllocation(); + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::DestroySignatureImpl(s64 offset, s64 size) { + /* Ensure we aren't already in a failed state. */ + R_TRY(m_last_result); + + /* Get our storage size. */ + s64 storage_size = 0; + R_TRY(this->UpdateLastResult(m_data_storage->GetSize(std::addressof(storage_size)))); + + /* Check the access range. */ + R_UNLESS(0 <= offset && offset < storage_size, fs::ResultInvalidOffset()); + + /* Determine the extents to clear signature for. */ + const auto start_offset = util::AlignUp(offset, m_verification_block_size); + const auto end_offset = util::AlignDown(std::min(offset + size, storage_size), m_verification_block_size); + + /* Flush the entries. */ + R_TRY(this->UpdateLastResult(this->FlushRangeCacheEntries(offset, size, true))); + + /* Clear the signature for the aligned range. */ + R_TRY(this->UpdateLastResult(m_data_storage->OperateRange(fs::OperationId::DestroySignature, start_offset, end_offset - start_offset))); + + /* Set blocking buffer manager allocations. */ + buffers::EnableBlockingBufferManagerAllocation(); + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::InvalidateImpl() { + /* Invalidate cache entries. */ + { + std::scoped_lock lk(*m_mutex); + + m_block_cache_manager.Invalidate(); + } + + /* Invalidate the aligned range. */ + { + Result result = m_data_storage->OperateRange(fs::OperationId::Invalidate, 0, std::numeric_limits::max()); + AMS_ASSERT(!fs::ResultBufferAllocationFailed::Includes(result)); + R_TRY(result); + } + + /* Clear our last result if we should. */ + if (fs::ResultIntegrityVerificationStorageCorrupted::Includes(m_last_result)) { + m_last_result = ResultSuccess(); + } + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::QueryRangeImpl(void *dst, size_t dst_size, s64 offset, s64 size) { + /* Get our storage size. */ + s64 storage_size = 0; + R_TRY(this->GetSize(std::addressof(storage_size))); + + /* Determine the extents we can actually query. */ + const auto actual_size = std::min(size, storage_size - offset); + const auto aligned_offset = util::AlignDown(offset, m_verification_block_size); + const auto aligned_offset_end = util::AlignUp(offset + actual_size, m_verification_block_size); + const auto aligned_size = aligned_offset_end - aligned_offset; + + /* Query the aligned range. */ + R_TRY(this->UpdateLastResult(m_data_storage->OperateRange(dst, dst_size, fs::OperationId::QueryRange, aligned_offset, aligned_size, nullptr, 0))); + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::GetAssociateBuffer(MemoryRange *out_range, CacheEntry *out_entry, s64 offset, size_t ideal_size, bool is_allocate_for_write) { + AMS_UNUSED(is_allocate_for_write); + + /* Validate pre-conditions. */ + AMS_ASSERT(m_data_storage != nullptr); + AMS_ASSERT(m_block_cache_manager.IsInitialized()); + AMS_ASSERT(out_range != nullptr); + AMS_ASSERT(out_entry != nullptr); + + /* Lock our mutex. */ + std::scoped_lock lk(*m_mutex); + + /* Get the maximum cache entry count. */ + const CacheIndex max_cache_entry_count = m_block_cache_manager.GetCount(); + + /* Locate the index of the cache entry, if present. */ + CacheIndex index; + size_t actual_size = ideal_size; + for (index = 0; index < max_cache_entry_count; ++index) { + if (const auto &entry = m_block_cache_manager[index]; entry.IsAllocated()) { + if (entry.range.IsIncluded(offset)) { + break; + } + + if (offset <= entry.range.offset && entry.range.offset < static_cast(offset + actual_size)) { + actual_size = static_cast(entry.range.offset - offset); + } + } + } + + /* Clear the out range. */ + out_range->first = 0; + out_range->second = 0; + + /* If we located an entry, use it. */ + if (index != max_cache_entry_count) { + m_block_cache_manager.AcquireCacheEntry(out_entry, out_range, index); + + actual_size = out_entry->range.size - (offset - out_entry->range.offset); + } + + /* If we don't have an out entry, allocate one. */ + if (out_range->first == 0) { + /* Ensure that the allocatable size is above a threshold. */ + const auto size_threshold = m_block_cache_manager.GetAllocator()->GetTotalSize() / 8; + if (m_block_cache_manager.GetAllocator()->GetTotalAllocatableSize() < size_threshold) { + R_TRY(this->FlushAllCacheEntries()); + } + + /* Decide in advance on a block alignment. */ + const size_t block_alignment = m_verification_block_size; + + /* Ensure that the size we request is valid. */ + { + AMS_ASSERT(actual_size >= 1); + actual_size = std::min(actual_size, block_alignment * 2); + } + AMS_ASSERT(actual_size >= block_alignment); + + /* Allocate a buffer. */ + R_TRY(buffers::AllocateBufferUsingBufferManagerContext(out_range, m_block_cache_manager.GetAllocator(), actual_size, fs::IBufferManager::BufferAttribute(m_buffer_level), [=](const MemoryRange &buffer) { + return buffer.first != 0 && block_alignment <= buffer.second; + }, AMS_CURRENT_FUNCTION_NAME)); + + /* Ensure our size is accurate. */ + actual_size = std::min(actual_size, out_range->second); + + /* Set the output entry. */ + out_entry->is_valid = true; + out_entry->is_write_back = false; + out_entry->is_cached = false; + out_entry->is_flushing = false; + out_entry->handle = 0; + out_entry->memory_address = 0; + out_entry->memory_size = 0; + out_entry->range.offset = offset; + out_entry->range.size = actual_size; + out_entry->lru_counter = 0; + } + + /* Check that we ended up with a coherent out range. */ + AMS_ASSERT(out_range->second >= out_entry->range.size); + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::StoreOrDestroyBuffer(CacheIndex *out, const MemoryRange &range, CacheEntry *entry) { + /* Validate pre-conditions. */ + AMS_ASSERT(out != nullptr); + + /* Lock our mutex. */ + std::scoped_lock lk(*m_mutex); + + /* In the event that we fail, release our buffer. */ + ON_RESULT_FAILURE { m_block_cache_manager.ReleaseCacheEntry(entry, range); }; + + /* If the entry is write-back, ensure we don't exceed certain dirtiness thresholds. */ + if (entry->is_write_back) { + R_TRY(this->ControlDirtiness()); + } + + /* Get unused cache entry index. */ + CacheIndex empty_index, lru_index; + m_block_cache_manager.GetEmptyCacheEntryIndex(std::addressof(empty_index), std::addressof(lru_index)); + + /* If all entries are valid, we need to invalidate one. */ + if (empty_index == BlockCacheManager::InvalidCacheIndex) { + /* Invalidate the lease recently used entry. */ + empty_index = lru_index; + + /* Get the entry to invalidate, sanity check that we can invalidate it. */ + const CacheEntry &entry_to_invalidate = m_block_cache_manager[empty_index]; + AMS_ASSERT(entry_to_invalidate.is_valid); + AMS_ASSERT(!entry_to_invalidate.is_flushing); + AMS_UNUSED(entry_to_invalidate); + + /* Invalidate the entry. */ + R_TRY(this->FlushCacheEntry(empty_index, true)); + + /* Check that the entry was invalidated successfully. */ + AMS_ASSERT(!entry_to_invalidate.is_valid); + AMS_ASSERT(!entry_to_invalidate.is_flushing); + } + + /* Store the entry. */ + if (m_block_cache_manager.SetCacheEntry(empty_index, *entry, range, fs::IBufferManager::BufferAttribute(m_buffer_level))) { + *out = empty_index; + } else { + *out = BlockCacheManager::InvalidCacheIndex; + } + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::FlushCacheEntry(CacheIndex index, bool invalidate) { + /* Lock our mutex. */ + std::scoped_lock lk(*m_mutex); + + /* Get the entry, sanity check that the entry's state allows for flush. */ + auto &entry = m_block_cache_manager[index]; + AMS_ASSERT(entry.is_valid); + AMS_ASSERT(!entry.is_flushing); + + /* If we're not write back (i.e. an invalidate is happening), just release the buffer. */ + if (!entry.is_write_back) { + AMS_ASSERT(invalidate); + + m_block_cache_manager.InvalidateCacheEntry(index); + + R_SUCCEED(); + } + + /* Note that we've started flushing, while we process. */ + m_block_cache_manager.SetFlushing(index, true); + ON_SCOPE_EXIT { m_block_cache_manager.SetFlushing(index, false); }; + + /* Create and check our memory range. */ + MemoryRange memory_range = fs::IBufferManager::MakeMemoryRange(entry.memory_address, entry.memory_size); + AMS_ASSERT(memory_range.first != 0); + AMS_ASSERT(memory_range.second >= entry.range.size); + + /* Validate the entry's offset. */ + AMS_ASSERT(entry.range.offset >= 0); + AMS_ASSERT(entry.range.offset < m_data_size); + AMS_ASSERT(util::IsAligned(entry.range.offset, m_verification_block_size)); + + /* Write back the data. */ + Result result = ResultSuccess(); + size_t write_size = entry.range.size; + if (R_SUCCEEDED(m_last_result)) { + /* Set blocking buffer manager allocations. */ + result = m_data_storage->Write(entry.range.offset, reinterpret_cast(memory_range.first), write_size); + + /* Check the result. */ + AMS_ASSERT(!fs::ResultBufferAllocationFailed::Includes(result)); + } else { + result = m_last_result; + } + + /* Set that we're not write-back. */ + m_block_cache_manager.SetWriteBack(index, false); + + /* If we're invalidating, release the buffer. Otherwise, register the flushed data. */ + if (invalidate) { + m_block_cache_manager.ReleaseCacheEntry(index, memory_range); + } else { + AMS_ASSERT(entry.is_valid); + m_block_cache_manager.RegisterCacheEntry(index, memory_range, fs::IBufferManager::BufferAttribute(m_buffer_level)); + } + + /* Try to succeed. */ + R_TRY(result); + + /* We succeeded. */ + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::FlushRangeCacheEntries(s64 offset, s64 size, bool invalidate) { + /* Validate pre-conditions. */ + AMS_ASSERT(m_data_storage != nullptr); + AMS_ASSERT(m_block_cache_manager.IsInitialized()); + + /* Iterate over all entries that fall within the range. */ + Result result = ResultSuccess(); + const auto max_cache_entry_count = m_block_cache_manager.GetCount(); + for (auto i = 0; i < max_cache_entry_count; ++i) { + auto &entry = m_block_cache_manager[i]; + if (entry.is_valid && (entry.is_write_back || invalidate) && (entry.range.offset < (offset + size)) && (offset < entry.range.GetEndOffset())) { + const auto cur_result = this->FlushCacheEntry(i, invalidate); + if (R_FAILED(cur_result) && R_SUCCEEDED(result)) { + result = cur_result; + } + } + } + + /* Try to succeed. */ + R_TRY(result); + + /* We succeeded. */ + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::FlushAllCacheEntries() { + R_TRY(this->FlushRangeCacheEntries(0, std::numeric_limits::max(), false)); + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::InvalidateAllCacheEntries() { + R_TRY(this->FlushRangeCacheEntries(0, std::numeric_limits::max(), true)); + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::ControlDirtiness() { + /* Get and validate the max cache entry count. */ + const auto max_cache_entry_count = m_block_cache_manager.GetCount(); + AMS_ASSERT(max_cache_entry_count > 0); + + /* Get size metrics from the buffer manager. */ + const auto total_size = m_block_cache_manager.GetAllocator()->GetTotalSize(); + const auto allocatable_size = m_block_cache_manager.GetAllocator()->GetTotalAllocatableSize(); + + /* If we have enough allocatable space, we don't need to do anything. */ + R_SUCCEED_IF(allocatable_size >= total_size / 4); + + /* Iterate over all entries (up to the threshold) and flush the least recently used dirty entry. */ + constexpr auto Threshold = 2; + for (int n = 0; n < Threshold; ++n) { + auto flushed_index = BlockCacheManager::InvalidCacheIndex; + for (auto index = 0; index < max_cache_entry_count; ++index) { + if (auto &entry = m_block_cache_manager[index]; entry.is_valid && entry.is_write_back) { + if (flushed_index == BlockCacheManager::InvalidCacheIndex || m_block_cache_manager[flushed_index].lru_counter < entry.lru_counter) { + flushed_index = index; + } + } + } + + /* If we can't flush anything, break. */ + if (flushed_index == BlockCacheManager::InvalidCacheIndex) { + break; + } + + R_TRY(this->FlushCacheEntry(flushed_index, false)); + } + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::UpdateLastResult(Result result) { + /* Update the last result. */ + if (R_FAILED(result) && !fs::ResultBufferAllocationFailed::Includes(result) && R_SUCCEEDED(m_last_result)) { + m_last_result = result; + } + + /* Try to succeed with the result. */ + R_TRY(result); + + /* We succeeded. */ + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::ReadHeadCache(MemoryRange *out_range, CacheEntry *out_entry, bool *out_cache_needed, s64 *offset, s64 *aligned_offset, s64 aligned_offset_end, char **buffer, size_t *size) { + /* Valdiate pre-conditions. */ + AMS_ASSERT(out_range != nullptr); + AMS_ASSERT(out_entry != nullptr); + AMS_ASSERT(out_cache_needed != nullptr); + AMS_ASSERT(offset != nullptr); + AMS_ASSERT(aligned_offset != nullptr); + AMS_ASSERT(buffer != nullptr); + AMS_ASSERT(*buffer != nullptr); + AMS_ASSERT(size != nullptr); + + AMS_ASSERT(*aligned_offset < aligned_offset_end); + + /* Iterate over the region. */ + CacheEntry entry = {}; + MemoryRange memory_range = {}; + *out_cache_needed = true; + + while (*aligned_offset < aligned_offset_end) { + /* Get the associated buffer for the offset. */ + R_TRY(this->UpdateLastResult(this->GetAssociateBuffer(std::addressof(memory_range), std::addressof(entry), *aligned_offset, m_verification_block_size, true))); + + /* If the entry isn't cached, we're done. */ + if (!entry.is_cached) { + break; + } + + /* Set cache not needed. */ + *out_cache_needed = false; + + /* Determine the size to copy. */ + const s64 buffer_offset = *offset - entry.range.offset; + const size_t copy_size = std::min(*size, static_cast(entry.range.GetEndOffset() - *offset)); + + /* Copy data from the entry. */ + std::memcpy(*buffer, reinterpret_cast(memory_range.first + buffer_offset), copy_size); + + /* Advance. */ + *buffer += copy_size; + *offset += copy_size; + *size -= copy_size; + *aligned_offset = entry.range.GetEndOffset(); + + /* Handle the buffer. */ + R_TRY(this->UpdateLastResult(this->StoreOrDestroyBuffer(memory_range, std::addressof(entry)))); + } + + /* Set the output entry. */ + *out_entry = entry; + *out_range = memory_range; + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::ReadTailCache(MemoryRange *out_range, CacheEntry *out_entry, bool *out_cache_needed, s64 offset, s64 aligned_offset, s64 *aligned_offset_end, char *buffer, size_t *size) { + /* Valdiate pre-conditions. */ + AMS_ASSERT(out_range != nullptr); + AMS_ASSERT(out_entry != nullptr); + AMS_ASSERT(out_cache_needed != nullptr); + AMS_ASSERT(aligned_offset_end != nullptr); + AMS_ASSERT(buffer != nullptr); + AMS_ASSERT(size != nullptr); + + AMS_ASSERT(aligned_offset < *aligned_offset_end); + + /* Iterate over the region. */ + CacheEntry entry = {}; + MemoryRange memory_range = {}; + *out_cache_needed = true; + + while (aligned_offset < *aligned_offset_end) { + /* Get the associated buffer for the offset. */ + R_TRY(this->UpdateLastResult(this->GetAssociateBuffer(std::addressof(memory_range), std::addressof(entry), *aligned_offset_end - m_verification_block_size, m_verification_block_size, true))); + + /* If the entry isn't cached, we're done. */ + if (!entry.is_cached) { + break; + } + + /* Set cache not needed. */ + *out_cache_needed = false; + + /* Determine the size to copy. */ + const s64 buffer_offset = std::max(static_cast(0), offset - entry.range.offset); + const size_t copy_size = std::min(*size, static_cast(offset + *size - entry.range.offset)); + + /* Copy data from the entry. */ + std::memcpy(buffer + *size - copy_size, reinterpret_cast(memory_range.first + buffer_offset), copy_size); + + /* Advance. */ + *size -= copy_size; + *aligned_offset_end = entry.range.offset; + + /* Handle the buffer. */ + R_TRY(this->UpdateLastResult(this->StoreOrDestroyBuffer(memory_range, std::addressof(entry)))); + } + + /* Set the output entry. */ + *out_entry = entry; + *out_range = memory_range; + + R_SUCCEED(); + } + + Result BlockCacheBufferedStorage::BulkRead(s64 offset, void *buffer, size_t size, MemoryRange *range_head, MemoryRange *range_tail, CacheEntry *entry_head, CacheEntry *entry_tail, bool head_cache_needed, bool tail_cache_needed) { + /* Validate pre-conditions. */ + AMS_ASSERT(buffer != nullptr); + AMS_ASSERT(range_head != nullptr); + AMS_ASSERT(range_tail != nullptr); + AMS_ASSERT(entry_head != nullptr); + AMS_ASSERT(entry_tail != nullptr); + + /* Determine bulk read offsets. */ + const s64 read_offset = offset; + const size_t read_size = size; + const s64 aligned_offset = util::AlignDown(read_offset, m_verification_block_size); + const s64 aligned_offset_end = util::AlignUp(read_offset + read_size, m_verification_block_size); + char *dst = static_cast(buffer); + + /* Prepare to do our reads. */ + auto head_guard = SCOPE_GUARD { m_block_cache_manager.ReleaseCacheEntry(entry_head, *range_head); }; + auto tail_guard = SCOPE_GUARD { m_block_cache_manager.ReleaseCacheEntry(entry_tail, *range_tail); }; + + /* Flush the entries. */ + R_TRY(this->UpdateLastResult(this->FlushRangeCacheEntries(aligned_offset, aligned_offset_end - aligned_offset, false))); + + /* Determine the buffer to read into. */ + PooledBuffer pooled_buffer; + const size_t buffer_size = static_cast(aligned_offset_end - aligned_offset); + char *read_buffer = nullptr; + if (read_offset == aligned_offset && read_size == buffer_size) { + read_buffer = dst; + } else if (tail_cache_needed && entry_tail->range.offset == aligned_offset && entry_tail->range.size == buffer_size) { + read_buffer = reinterpret_cast(range_tail->first); + } else if (head_cache_needed && entry_head->range.offset == aligned_offset && entry_head->range.size == buffer_size) { + read_buffer = reinterpret_cast(range_head->first); + } else { + pooled_buffer.AllocateParticularlyLarge(buffer_size, 1); + R_UNLESS(pooled_buffer.GetSize() >= buffer_size, fs::ResultAllocationFailurePooledBufferNotEnoughSize()); + read_buffer = pooled_buffer.GetBuffer(); + } + + /* Read the data. */ + R_TRY(m_data_storage->Read(aligned_offset, read_buffer, buffer_size)); + + /* Copy the data out. */ + if (dst != read_buffer) { + std::memcpy(dst, read_buffer + read_offset - aligned_offset, read_size); + } + + /* Create a helper to populate our caches. */ + const auto PopulateCacheFromPooledBuffer = [&](CacheEntry *entry, MemoryRange *range) { + AMS_ASSERT(entry != nullptr); + AMS_ASSERT(range != nullptr); + + if (aligned_offset <= entry->range.offset && entry->range.GetEndOffset() <= static_cast(aligned_offset + buffer_size)) { + AMS_ASSERT(!entry->is_cached); + if (reinterpret_cast(range->first) != read_buffer) { + std::memcpy(reinterpret_cast(range->first), read_buffer + entry->range.offset - aligned_offset, entry->range.size); + } + entry->is_cached = true; + } + }; + + /* Populate tail cache if needed. */ + if (tail_cache_needed) { + PopulateCacheFromPooledBuffer(entry_tail, range_tail); + } + + /* Populate head cache if needed. */ + if (head_cache_needed) { + PopulateCacheFromPooledBuffer(entry_head, range_head); + } + + /* If both entries are cached, one may contain the other; in that case, we need only the larger entry. */ + if (entry_head->is_cached && entry_tail->is_cached) { + if (entry_tail->range.offset <= entry_head->range.offset && entry_head->range.GetEndOffset() <= entry_tail->range.GetEndOffset()) { + entry_head->is_cached = false; + } else if (entry_head->range.offset <= entry_tail->range.offset && entry_tail->range.GetEndOffset() <= entry_head->range.GetEndOffset()) { + entry_tail->is_cached = false; + } + } + + /* Destroy the tail cache. */ + tail_guard.Cancel(); + if (entry_tail->is_cached) { + R_TRY(this->UpdateLastResult(this->StoreOrDestroyBuffer(*range_tail, entry_tail))); + } else { + m_block_cache_manager.ReleaseCacheEntry(entry_tail, *range_tail); + } + + /* Destroy the head cache. */ + head_guard.Cancel(); + if (entry_head->is_cached) { + R_TRY(this->UpdateLastResult(this->StoreOrDestroyBuffer(*range_head, entry_head))); + } else { + m_block_cache_manager.ReleaseCacheEntry(entry_head, *range_head); + } + + R_SUCCEED(); + } + +} diff --git a/libraries/libstratosphere/source/fssystem/fssystem_buffered_storage.cpp b/libraries/libstratosphere/source/fssystem/fssystem_buffered_storage.cpp new file mode 100644 index 0000000..96433aa --- /dev/null +++ b/libraries/libstratosphere/source/fssystem/fssystem_buffered_storage.cpp @@ -0,0 +1,1085 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::fssystem { + + namespace { + + constexpr inline uintptr_t InvalidAddress = 0; + constexpr inline s64 InvalidOffset = std::numeric_limits::max(); + + } + + class BufferedStorage::Cache : public ::ams::fs::impl::Newable { + private: + struct FetchParameter { + s64 offset; + void *buffer; + size_t size; + }; + static_assert(util::is_pod::value); + private: + BufferedStorage *m_buffered_storage; + std::pair m_memory_range; + fs::IBufferManager::CacheHandle m_cache_handle; + s64 m_offset; + std::atomic m_is_valid; + std::atomic m_is_dirty; + u8 m_reserved[2]; + s32 m_reference_count; + Cache *m_next; + Cache *m_prev; + public: + Cache() : m_buffered_storage(nullptr), m_memory_range(InvalidAddress, 0), m_cache_handle(), m_offset(InvalidOffset), m_is_valid(false), m_is_dirty(false), m_reference_count(1), m_next(nullptr), m_prev(nullptr) { + /* ... */ + } + + ~Cache() { + this->Finalize(); + } + + void Initialize(BufferedStorage *bs) { + AMS_ASSERT(bs != nullptr); + AMS_ASSERT(m_buffered_storage == nullptr); + + m_buffered_storage = bs; + this->Link(); + } + + void Finalize() { + AMS_ASSERT(m_buffered_storage != nullptr); + AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); + AMS_ASSERT(m_reference_count == 0); + + /* If we're valid, acquire our cache handle and free our buffer. */ + if (this->IsValid()) { + const auto buffer_manager = m_buffered_storage->m_buffer_manager; + if (!m_is_dirty) { + AMS_ASSERT(m_memory_range.first == InvalidAddress); + m_memory_range = buffer_manager->AcquireCache(m_cache_handle); + } + if (m_memory_range.first != InvalidAddress) { + buffer_manager->DeallocateBuffer(m_memory_range.first, m_memory_range.second); + m_memory_range.first = InvalidAddress; + m_memory_range.second = 0; + } + } + + /* Clear all our members. */ + m_buffered_storage = nullptr; + m_offset = InvalidOffset; + m_is_valid = false; + m_is_dirty = false; + m_next = nullptr; + m_prev = nullptr; + } + + void Link() { + AMS_ASSERT(m_buffered_storage != nullptr); + AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); + AMS_ASSERT(m_reference_count > 0); + + if ((--m_reference_count) == 0) { + AMS_ASSERT(m_next == nullptr); + AMS_ASSERT(m_prev == nullptr); + + if (m_buffered_storage->m_next_fetch_cache == nullptr) { + m_buffered_storage->m_next_fetch_cache = this; + m_next = this; + m_prev = this; + } else { + /* Check against a cache being registered twice. */ + { + auto cache = m_buffered_storage->m_next_fetch_cache; + do { + if (cache->IsValid() && this->Hits(cache->m_offset, m_buffered_storage->m_block_size)) { + m_is_valid = false; + break; + } + cache = cache->m_next; + } while (cache != m_buffered_storage->m_next_fetch_cache); + } + + /* Link into the fetch list. */ + { + AMS_ASSERT(m_buffered_storage->m_next_fetch_cache->m_prev != nullptr); + AMS_ASSERT(m_buffered_storage->m_next_fetch_cache->m_prev->m_next == m_buffered_storage->m_next_fetch_cache); + m_next = m_buffered_storage->m_next_fetch_cache; + m_prev = m_buffered_storage->m_next_fetch_cache->m_prev; + m_next->m_prev = this; + m_prev->m_next = this; + } + + /* Insert invalid caches at the start of the list. */ + if (!this->IsValid()) { + m_buffered_storage->m_next_fetch_cache = this; + } + } + + /* If we're not valid, clear our offset. */ + if (!this->IsValid()) { + m_offset = InvalidOffset; + m_is_dirty = false; + } + + /* Ensure our buffer state is coherent. */ + if (m_memory_range.first != InvalidAddress && !m_is_dirty) { + if (this->IsValid()) { + m_cache_handle = m_buffered_storage->m_buffer_manager->RegisterCache(m_memory_range.first, m_memory_range.second, fs::IBufferManager::BufferAttribute()); + } else { + m_buffered_storage->m_buffer_manager->DeallocateBuffer(m_memory_range.first, m_memory_range.second); + } + m_memory_range.first = InvalidAddress; + m_memory_range.second = 0; + } + } + } + + void Unlink() { + AMS_ASSERT(m_buffered_storage != nullptr); + AMS_ASSERT(m_reference_count >= 0); + + if ((++m_reference_count) == 1) { + AMS_ASSERT(m_next != nullptr); + AMS_ASSERT(m_prev != nullptr); + AMS_ASSERT(m_next->m_prev == this); + AMS_ASSERT(m_prev->m_next == this); + + if (m_buffered_storage->m_next_fetch_cache == this) { + if (m_next != this) { + m_buffered_storage->m_next_fetch_cache = m_next; + } else { + m_buffered_storage->m_next_fetch_cache = nullptr; + } + } + + m_buffered_storage->m_next_acquire_cache = this; + + m_next->m_prev = m_prev; + m_prev->m_next = m_next; + m_next = nullptr; + m_prev = nullptr; + } else { + AMS_ASSERT(m_next == nullptr); + AMS_ASSERT(m_prev == nullptr); + } + } + + void Read(s64 offset, void *buffer, size_t size) const { + AMS_ASSERT(m_buffered_storage != nullptr); + AMS_ASSERT(m_next == nullptr); + AMS_ASSERT(m_prev == nullptr); + AMS_ASSERT(this->IsValid()); + AMS_ASSERT(this->Hits(offset, 1)); + AMS_ASSERT(m_memory_range.first != InvalidAddress); + + const auto read_offset = offset - m_offset; + const auto readable_offset_max = m_buffered_storage->m_block_size - size; + const auto cache_buffer = reinterpret_cast(m_memory_range.first) + read_offset; + AMS_ASSERT(read_offset >= 0); + AMS_ASSERT(static_cast(read_offset) <= readable_offset_max); + AMS_UNUSED(readable_offset_max); + + std::memcpy(buffer, cache_buffer, size); + } + + void Write(s64 offset, const void *buffer, size_t size) { + AMS_ASSERT(m_buffered_storage != nullptr); + AMS_ASSERT(m_next == nullptr); + AMS_ASSERT(m_prev == nullptr); + AMS_ASSERT(this->IsValid()); + AMS_ASSERT(this->Hits(offset, 1)); + AMS_ASSERT(m_memory_range.first != InvalidAddress); + + const auto write_offset = offset - m_offset; + const auto writable_offset_max = m_buffered_storage->m_block_size - size; + const auto cache_buffer = reinterpret_cast(m_memory_range.first) + write_offset; + AMS_ASSERT(write_offset >= 0); + AMS_ASSERT(static_cast(write_offset) <= writable_offset_max); + AMS_UNUSED(writable_offset_max); + + std::memcpy(cache_buffer, buffer, size); + m_is_dirty = true; + } + + Result Flush() { + AMS_ASSERT(m_buffered_storage != nullptr); + AMS_ASSERT(m_next == nullptr); + AMS_ASSERT(m_prev == nullptr); + AMS_ASSERT(this->IsValid()); + + if (m_is_dirty) { + AMS_ASSERT(m_memory_range.first != InvalidAddress); + + const auto base_size = m_buffered_storage->m_base_storage_size; + const auto block_size = static_cast(m_buffered_storage->m_block_size); + const auto flush_size = static_cast(std::min(block_size, base_size - m_offset)); + + auto &base_storage = m_buffered_storage->m_base_storage; + const auto cache_buffer = reinterpret_cast(m_memory_range.first); + + R_TRY(base_storage.Write(m_offset, cache_buffer, flush_size)); + m_is_dirty = false; + + buffers::EnableBlockingBufferManagerAllocation(); + } + + return ResultSuccess(); + } + + const std::pair PrepareFetch() { + AMS_ASSERT(m_buffered_storage != nullptr); + AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); + AMS_ASSERT(m_next == nullptr); + AMS_ASSERT(m_prev == nullptr); + AMS_ASSERT(this->IsValid()); + AMS_ASSERT(m_buffered_storage->m_mutex.IsLockedByCurrentThread()); + + std::pair result(ResultSuccess(), false); + if (m_reference_count == 1) { + result.first = this->Flush(); + if (R_SUCCEEDED(result.first)) { + m_is_valid = false; + m_reference_count = 0; + result.second = true; + } + } + + return result; + } + + void UnprepareFetch() { + AMS_ASSERT(m_buffered_storage != nullptr); + AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); + AMS_ASSERT(m_next == nullptr); + AMS_ASSERT(m_prev == nullptr); + AMS_ASSERT(!this->IsValid()); + AMS_ASSERT(!m_is_dirty); + AMS_ASSERT(m_buffered_storage->m_mutex.IsLockedByCurrentThread()); + + m_is_valid = true; + m_reference_count = 1; + } + + Result Fetch(s64 offset) { + AMS_ASSERT(m_buffered_storage != nullptr); + AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); + AMS_ASSERT(m_next == nullptr); + AMS_ASSERT(m_prev == nullptr); + AMS_ASSERT(!this->IsValid()); + AMS_ASSERT(!m_is_dirty); + + if (m_memory_range.first == InvalidAddress) { + R_TRY(this->AllocateFetchBuffer()); + } + + FetchParameter fetch_param = {}; + this->CalcFetchParameter(std::addressof(fetch_param), offset); + + auto &base_storage = m_buffered_storage->m_base_storage; + R_TRY(base_storage.Read(fetch_param.offset, fetch_param.buffer, fetch_param.size)); + m_offset = fetch_param.offset; + AMS_ASSERT(this->Hits(offset, 1)); + + return ResultSuccess(); + } + + Result FetchFromBuffer(s64 offset, const void *buffer, size_t buffer_size) { + AMS_ASSERT(m_buffered_storage != nullptr); + AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); + AMS_ASSERT(m_next == nullptr); + AMS_ASSERT(m_prev == nullptr); + AMS_ASSERT(!this->IsValid()); + AMS_ASSERT(!m_is_dirty); + AMS_ASSERT(util::IsAligned(offset, m_buffered_storage->m_block_size)); + + if (m_memory_range.first == InvalidAddress) { + R_TRY(this->AllocateFetchBuffer()); + } + + FetchParameter fetch_param = {}; + this->CalcFetchParameter(std::addressof(fetch_param), offset); + AMS_ASSERT(fetch_param.offset == offset); + AMS_ASSERT(fetch_param.size <= buffer_size); + AMS_UNUSED(buffer_size); + + std::memcpy(fetch_param.buffer, buffer, fetch_param.size); + m_offset = fetch_param.offset; + AMS_ASSERT(this->Hits(offset, 1)); + + return ResultSuccess(); + } + + bool TryAcquireCache() { + AMS_ASSERT(m_buffered_storage != nullptr); + AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); + AMS_ASSERT(this->IsValid()); + + if (m_memory_range.first != InvalidAddress) { + return true; + } else { + m_memory_range = m_buffered_storage->m_buffer_manager->AcquireCache(m_cache_handle); + m_is_valid = m_memory_range.first != InvalidAddress; + return m_is_valid; + } + } + + void Invalidate() { + AMS_ASSERT(m_buffered_storage != nullptr); + m_is_valid = false; + } + + bool IsValid() const { + AMS_ASSERT(m_buffered_storage != nullptr); + return m_is_valid || m_reference_count > 0; + } + + bool IsDirty() const { + AMS_ASSERT(m_buffered_storage != nullptr); + return m_is_dirty; + } + + bool Hits(s64 offset, s64 size) const { + AMS_ASSERT(m_buffered_storage != nullptr); + const auto block_size = static_cast(m_buffered_storage->m_block_size); + return (offset < m_offset + block_size) && (m_offset < offset + size); + } + private: + Result AllocateFetchBuffer() { + fs::IBufferManager *buffer_manager = m_buffered_storage->m_buffer_manager; + AMS_ASSERT(buffer_manager->AcquireCache(m_cache_handle).first == InvalidAddress); + + auto range_guard = SCOPE_GUARD { m_memory_range.first = InvalidAddress; }; + R_TRY(buffers::AllocateBufferUsingBufferManagerContext(std::addressof(m_memory_range), buffer_manager, m_buffered_storage->m_block_size, fs::IBufferManager::BufferAttribute(), [](const std::pair &buffer) { + return buffer.first != 0; + }, AMS_CURRENT_FUNCTION_NAME)); + + range_guard.Cancel(); + return ResultSuccess(); + } + + void CalcFetchParameter(FetchParameter *out, s64 offset) const { + AMS_ASSERT(out != nullptr); + + const auto block_size = static_cast(m_buffered_storage->m_block_size); + const auto cache_offset = util::AlignDown(offset, m_buffered_storage->m_block_size); + const auto base_size = m_buffered_storage->m_base_storage_size; + const auto cache_size = static_cast(std::min(block_size, base_size - cache_offset)); + const auto cache_buffer = reinterpret_cast(m_memory_range.first); + AMS_ASSERT(offset >= 0); + AMS_ASSERT(offset < base_size); + + out->offset = cache_offset; + out->buffer = cache_buffer; + out->size = cache_size; + } + }; + + class BufferedStorage::SharedCache { + NON_COPYABLE(SharedCache); + NON_MOVEABLE(SharedCache); + friend class UniqueCache; + private: + Cache *m_cache; + Cache *m_start_cache; + BufferedStorage *m_buffered_storage; + public: + explicit SharedCache(BufferedStorage *bs) : m_cache(nullptr), m_start_cache(bs->m_next_acquire_cache), m_buffered_storage(bs) { + AMS_ASSERT(m_buffered_storage != nullptr); + } + + ~SharedCache() { + std::scoped_lock lk(m_buffered_storage->m_mutex); + this->Release(); + } + + bool AcquireNextOverlappedCache(s64 offset, s64 size) { + AMS_ASSERT(m_buffered_storage != nullptr); + + auto is_first = m_cache == nullptr; + const auto start = is_first ? m_start_cache : m_cache + 1; + + AMS_ASSERT(start >= m_buffered_storage->m_caches.get()); + AMS_ASSERT(start <= m_buffered_storage->m_caches.get() + m_buffered_storage->m_cache_count); + + std::scoped_lock lk(m_buffered_storage->m_mutex); + + this->Release(); + AMS_ASSERT(m_cache == nullptr); + + for (auto cache = start; true; ++cache) { + if (m_buffered_storage->m_caches.get() + m_buffered_storage->m_cache_count <= cache) { + cache = m_buffered_storage->m_caches.get(); + } + if (!is_first && cache == m_start_cache) { + break; + } + if (cache->IsValid() && cache->Hits(offset, size) && cache->TryAcquireCache()) { + cache->Unlink(); + m_cache = cache; + return true; + } + is_first = false; + } + + m_cache = nullptr; + return false; + } + + bool AcquireNextDirtyCache() { + AMS_ASSERT(m_buffered_storage != nullptr); + const auto start = m_cache != nullptr ? m_cache + 1 : m_buffered_storage->m_caches.get(); + const auto end = m_buffered_storage->m_caches.get() + m_buffered_storage->m_cache_count; + + AMS_ASSERT(start >= m_buffered_storage->m_caches.get()); + AMS_ASSERT(start <= end); + + this->Release(); + AMS_ASSERT(m_cache == nullptr); + + for (auto cache = start; cache < end; ++cache) { + if (cache->IsValid() && cache->IsDirty() && cache->TryAcquireCache()) { + cache->Unlink(); + m_cache = cache; + return true; + } + } + + m_cache = nullptr; + return false; + } + + bool AcquireNextValidCache() { + AMS_ASSERT(m_buffered_storage != nullptr); + const auto start = m_cache != nullptr ? m_cache + 1 : m_buffered_storage->m_caches.get(); + const auto end = m_buffered_storage->m_caches.get() + m_buffered_storage->m_cache_count; + + AMS_ASSERT(start >= m_buffered_storage->m_caches.get()); + AMS_ASSERT(start <= end); + + this->Release(); + AMS_ASSERT(m_cache == nullptr); + + for (auto cache = start; cache < end; ++cache) { + if (cache->IsValid() && cache->TryAcquireCache()) { + cache->Unlink(); + m_cache = cache; + return true; + } + } + + m_cache = nullptr; + return false; + } + + bool AcquireFetchableCache() { + AMS_ASSERT(m_buffered_storage != nullptr); + + std::scoped_lock lk(m_buffered_storage->m_mutex); + + this->Release(); + AMS_ASSERT(m_cache == nullptr); + + m_cache = m_buffered_storage->m_next_fetch_cache; + if (m_cache != nullptr) { + if (m_cache->IsValid()) { + m_cache->TryAcquireCache(); + } + m_cache->Unlink(); + } + + return m_cache != nullptr; + } + + void Read(s64 offset, void *buffer, size_t size) { + AMS_ASSERT(m_cache != nullptr); + m_cache->Read(offset, buffer, size); + } + + void Write(s64 offset, const void *buffer, size_t size) { + AMS_ASSERT(m_cache != nullptr); + m_cache->Write(offset, buffer, size); + } + + Result Flush() { + AMS_ASSERT(m_cache != nullptr); + return m_cache->Flush(); + } + + void Invalidate() { + AMS_ASSERT(m_cache != nullptr); + return m_cache->Invalidate(); + } + + bool Hits(s64 offset, s64 size) const { + AMS_ASSERT(m_cache != nullptr); + return m_cache->Hits(offset, size); + } + private: + void Release() { + if (m_cache != nullptr) { + AMS_ASSERT(m_buffered_storage->m_caches.get() <= m_cache); + AMS_ASSERT(m_cache <= m_buffered_storage->m_caches.get() + m_buffered_storage->m_cache_count); + + m_cache->Link(); + m_cache = nullptr; + } + } + }; + + class BufferedStorage::UniqueCache { + NON_COPYABLE(UniqueCache); + NON_MOVEABLE(UniqueCache); + private: + Cache *m_cache; + BufferedStorage *m_buffered_storage; + public: + explicit UniqueCache(BufferedStorage *bs) : m_cache(nullptr), m_buffered_storage(bs) { + AMS_ASSERT(m_buffered_storage != nullptr); + } + + ~UniqueCache() { + if (m_cache != nullptr) { + std::scoped_lock lk(m_buffered_storage->m_mutex); + m_cache->UnprepareFetch(); + } + } + + const std::pair Upgrade(const SharedCache &shared_cache) { + AMS_ASSERT(m_buffered_storage == shared_cache.m_buffered_storage); + AMS_ASSERT(shared_cache.m_cache != nullptr); + + std::scoped_lock lk(m_buffered_storage->m_mutex); + const auto result = shared_cache.m_cache->PrepareFetch(); + if (R_SUCCEEDED(result.first) && result.second) { + m_cache = shared_cache.m_cache; + } + return result; + } + + Result Fetch(s64 offset) { + AMS_ASSERT(m_cache != nullptr); + return m_cache->Fetch(offset); + } + + Result FetchFromBuffer(s64 offset, const void *buffer, size_t buffer_size) { + AMS_ASSERT(m_cache != nullptr); + R_TRY(m_cache->FetchFromBuffer(offset, buffer, buffer_size)); + return ResultSuccess(); + } + }; + + BufferedStorage::BufferedStorage() : m_base_storage(), m_buffer_manager(), m_block_size(), m_base_storage_size(), m_caches(), m_cache_count(), m_next_acquire_cache(), m_next_fetch_cache(), m_mutex(), m_bulk_read_enabled() { + /* ... */ + } + + BufferedStorage::~BufferedStorage() { + this->Finalize(); + } + + Result BufferedStorage::Initialize(fs::SubStorage base_storage, fs::IBufferManager *buffer_manager, size_t block_size, s32 buffer_count) { + AMS_ASSERT(buffer_manager != nullptr); + AMS_ASSERT(block_size > 0); + AMS_ASSERT(util::IsPowerOfTwo(block_size)); + AMS_ASSERT(buffer_count > 0); + + /* Get the base storage size. */ + R_TRY(base_storage.GetSize(std::addressof(m_base_storage_size))); + + /* Set members. */ + m_base_storage = base_storage; + m_buffer_manager = buffer_manager; + m_block_size = block_size; + m_cache_count = buffer_count; + + /* Allocate the caches. */ + m_caches.reset(new Cache[buffer_count]); + R_UNLESS(m_caches != nullptr, fs::ResultAllocationFailureInBufferedStorageA()); + + /* Initialize the caches. */ + for (auto i = 0; i < buffer_count; i++) { + m_caches[i].Initialize(this); + } + + m_next_acquire_cache = std::addressof(m_caches[0]); + return ResultSuccess(); + } + + void BufferedStorage::Finalize() { + m_base_storage = fs::SubStorage(); + m_base_storage_size = 0; + m_caches.reset(); + m_cache_count = 0; + m_next_fetch_cache = nullptr; + } + + Result BufferedStorage::Read(s64 offset, void *buffer, size_t size) { + AMS_ASSERT(this->IsInitialized()); + + /* Succeed if zero size. */ + R_SUCCEED_IF(size == 0); + + /* Validate arguments. */ + R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); + + /* Do the read. */ + R_TRY(this->ReadCore(offset, buffer, size)); + return ResultSuccess(); + } + + Result BufferedStorage::Write(s64 offset, const void *buffer, size_t size) { + AMS_ASSERT(this->IsInitialized()); + + /* Succeed if zero size. */ + R_SUCCEED_IF(size == 0); + + /* Validate arguments. */ + R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); + + /* Do the write. */ + R_TRY(this->WriteCore(offset, buffer, size)); + return ResultSuccess(); + } + + Result BufferedStorage::GetSize(s64 *out) { + AMS_ASSERT(out != nullptr); + AMS_ASSERT(this->IsInitialized()); + + *out = m_base_storage_size; + return ResultSuccess(); + } + + Result BufferedStorage::SetSize(s64 size) { + AMS_ASSERT(this->IsInitialized()); + const s64 prev_size = m_base_storage_size; + if (prev_size < size) { + /* Prepare to expand. */ + if (!util::IsAligned(prev_size, m_block_size)) { + SharedCache cache(this); + const auto invalidate_offset = prev_size; + const auto invalidate_size = size - prev_size; + if (cache.AcquireNextOverlappedCache(invalidate_offset, invalidate_size)) { + R_TRY(cache.Flush()); + cache.Invalidate(); + } + AMS_ASSERT(!cache.AcquireNextOverlappedCache(invalidate_offset, invalidate_size)); + } + } else if (size < prev_size) { + /* Prepare to do a shrink. */ + SharedCache cache(this); + const auto invalidate_offset = prev_size; + const auto invalidate_size = size - prev_size; + const auto is_fragment = util::IsAligned(size, m_block_size); + while (cache.AcquireNextOverlappedCache(invalidate_offset, invalidate_size)) { + if (is_fragment && cache.Hits(invalidate_offset, 1)) { + R_TRY(cache.Flush()); + } + cache.Invalidate(); + } + } + + /* Set the size. */ + R_TRY(m_base_storage.SetSize(size)); + + /* Get our new size. */ + s64 new_size = 0; + R_TRY(m_base_storage.GetSize(std::addressof(new_size))); + + m_base_storage_size = new_size; + return ResultSuccess(); + } + + Result BufferedStorage::Flush() { + AMS_ASSERT(this->IsInitialized()); + + /* Flush caches. */ + SharedCache cache(this); + while (cache.AcquireNextDirtyCache()) { + R_TRY(cache.Flush()); + } + + /* Flush the base storage. */ + R_TRY(m_base_storage.Flush()); + return ResultSuccess(); + } + + Result BufferedStorage::OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) { + AMS_ASSERT(this->IsInitialized()); + + /* Invalidate caches, if we should. */ + if (op_id == fs::OperationId::Invalidate) { + SharedCache cache(this); + while (cache.AcquireNextOverlappedCache(offset, size)) { + cache.Invalidate(); + } + } + + return m_base_storage.OperateRange(dst, dst_size, op_id, offset, size, src, src_size); + } + + void BufferedStorage::InvalidateCaches() { + AMS_ASSERT(this->IsInitialized()); + + SharedCache cache(this); + while (cache.AcquireNextValidCache()) { + cache.Invalidate(); + } + } + + Result BufferedStorage::PrepareAllocation() { + const auto flush_threshold = m_buffer_manager->GetTotalSize() / 8; + if (m_buffer_manager->GetTotalAllocatableSize() < flush_threshold) { + R_TRY(this->Flush()); + } + return ResultSuccess(); + } + + Result BufferedStorage::ControlDirtiness() { + const auto flush_threshold = m_buffer_manager->GetTotalSize() / 4; + if (m_buffer_manager->GetTotalAllocatableSize() < flush_threshold) { + s32 dirty_count = 0; + SharedCache cache(this); + while (cache.AcquireNextDirtyCache()) { + if ((++dirty_count) > 1) { + R_TRY(cache.Flush()); + cache.Invalidate(); + } + } + } + return ResultSuccess(); + } + + Result BufferedStorage::ReadCore(s64 offset, void *buffer, size_t size) { + AMS_ASSERT(m_caches != nullptr); + AMS_ASSERT(buffer != nullptr); + + /* Validate the offset. */ + const auto base_storage_size = m_base_storage_size; + R_UNLESS(offset >= 0, fs::ResultInvalidOffset()); + R_UNLESS(offset <= base_storage_size, fs::ResultInvalidOffset()); + + /* Setup tracking variables. */ + size_t remaining_size = static_cast(std::min(size, base_storage_size - offset)); + s64 cur_offset = offset; + s64 buf_offset = 0; + + /* Determine what caches are needed, if we have bulk read set. */ + if (m_bulk_read_enabled) { + /* Check head cache. */ + const auto head_cache_needed = this->ReadHeadCache(std::addressof(cur_offset), buffer, std::addressof(remaining_size), std::addressof(buf_offset)); + R_SUCCEED_IF(remaining_size == 0); + + /* Check tail cache. */ + const auto tail_cache_needed = this->ReadTailCache(cur_offset, buffer, std::addressof(remaining_size), buf_offset); + R_SUCCEED_IF(remaining_size == 0); + + /* Perform bulk reads. */ + constexpr size_t BulkReadSizeMax = 2_MB; + if (remaining_size <= BulkReadSizeMax) { + do { + /* Try to do a bulk read. */ + R_TRY_CATCH(this->BulkRead(cur_offset, static_cast(buffer) + buf_offset, remaining_size, head_cache_needed, tail_cache_needed)) { + R_CATCH(fs::ResultAllocationFailurePooledBufferNotEnoughSize) { + /* If the read fails due to insufficient pooled buffer size, */ + /* then we want to fall back to the normal read path. */ + break; + } + } R_END_TRY_CATCH; + + return ResultSuccess(); + } while(0); + } + } + + /* Repeatedly read until we're done. */ + while (remaining_size > 0) { + /* Determine how much to read this iteration. */ + auto *cur_dst = static_cast(buffer) + buf_offset; + size_t cur_size = 0; + + if (!util::IsAligned(cur_offset, m_block_size)) { + const size_t aligned_size = m_block_size - (cur_offset & (m_block_size - 1)); + cur_size = std::min(aligned_size, remaining_size); + } else if (remaining_size < m_block_size) { + cur_size = remaining_size; + } else { + cur_size = util::AlignDown(remaining_size, m_block_size); + } + + if (cur_size <= m_block_size) { + SharedCache cache(this); + if (!cache.AcquireNextOverlappedCache(cur_offset, cur_size)) { + R_TRY(this->PrepareAllocation()); + while (true) { + R_UNLESS(cache.AcquireFetchableCache(), fs::ResultOutOfResource()); + + UniqueCache fetch_cache(this); + const auto upgrade_result = fetch_cache.Upgrade(cache); + R_TRY(upgrade_result.first); + if (upgrade_result.second) { + R_TRY(fetch_cache.Fetch(cur_offset)); + break; + } + } + R_TRY(this->ControlDirtiness()); + } + cache.Read(cur_offset, cur_dst, cur_size); + } else { + { + SharedCache cache(this); + while (cache.AcquireNextOverlappedCache(cur_offset, cur_size)) { + R_TRY(cache.Flush()); + cache.Invalidate(); + } + } + R_TRY(m_base_storage.Read(cur_offset, cur_dst, cur_size)); + } + + remaining_size -= cur_size; + cur_offset += cur_size; + buf_offset += cur_size; + } + + return ResultSuccess(); + } + + bool BufferedStorage::ReadHeadCache(s64 *offset, void *buffer, size_t *size, s64 *buffer_offset) { + AMS_ASSERT(offset != nullptr); + AMS_ASSERT(buffer != nullptr); + AMS_ASSERT(size != nullptr); + AMS_ASSERT(buffer_offset != nullptr); + + bool is_cache_needed = !util::IsAligned(*offset, m_block_size); + + while (*size > 0) { + size_t cur_size = 0; + + if (!util::IsAligned(*offset, m_block_size)) { + const s64 aligned_size = util::AlignUp(*offset, m_block_size) - *offset; + cur_size = std::min(aligned_size, static_cast(*size)); + } else if (*size < m_block_size) { + cur_size = *size; + } else { + cur_size = m_block_size; + } + + SharedCache cache(this); + if (!cache.AcquireNextOverlappedCache(*offset, cur_size)) { + break; + } + + cache.Read(*offset, static_cast(buffer) + *buffer_offset, cur_size); + *offset += cur_size; + *buffer_offset += cur_size; + *size -= cur_size; + is_cache_needed = false; + } + + return is_cache_needed; + } + + bool BufferedStorage::ReadTailCache(s64 offset, void *buffer, size_t *size, s64 buffer_offset) { + AMS_ASSERT(buffer != nullptr); + AMS_ASSERT(size != nullptr); + + bool is_cache_needed = !util::IsAligned(offset + *size, m_block_size); + + while (*size > 0) { + const s64 cur_offset_end = offset + *size; + size_t cur_size = 0; + + if (!util::IsAligned(cur_offset_end, m_block_size)) { + const s64 aligned_size = cur_offset_end - util::AlignDown(cur_offset_end, m_block_size); + cur_size = std::min(aligned_size, static_cast(*size)); + } else if (*size < m_block_size) { + cur_size = *size; + } else { + cur_size = m_block_size; + } + + const s64 cur_offset = cur_offset_end - static_cast(cur_size); + AMS_ASSERT(cur_offset >= 0); + + SharedCache cache(this); + if (!cache.AcquireNextOverlappedCache(cur_offset, cur_size)) { + break; + } + + cache.Read(cur_offset, static_cast(buffer) + buffer_offset + cur_offset - offset, cur_size); + *size -= cur_size; + is_cache_needed = false; + } + + return is_cache_needed; + } + + Result BufferedStorage::BulkRead(s64 offset, void *buffer, size_t size, bool head_cache_needed, bool tail_cache_needed) { + /* Determine aligned extents. */ + const s64 aligned_offset = util::AlignDown(offset, m_block_size); + const s64 aligned_offset_end = std::min(util::AlignUp(offset + static_cast(size), m_block_size), m_base_storage_size); + const s64 aligned_size = aligned_offset_end - aligned_offset; + + /* Allocate a work buffer. */ + char *work_buffer = nullptr; + PooledBuffer pooled_buffer; + if (offset == aligned_offset && size == static_cast(aligned_size)) { + work_buffer = static_cast(buffer); + } else { + pooled_buffer.AllocateParticularlyLarge(static_cast(aligned_size), 1); + R_UNLESS(static_cast(pooled_buffer.GetSize()) >= aligned_size, fs::ResultAllocationFailurePooledBufferNotEnoughSize()); + work_buffer = pooled_buffer.GetBuffer(); + } + + /* Ensure cache is coherent. */ + { + SharedCache cache(this); + while (cache.AcquireNextOverlappedCache(aligned_offset, aligned_size)) { + R_TRY(cache.Flush()); + cache.Invalidate(); + } + } + + /* Read from the base storage. */ + R_TRY(m_base_storage.Read(aligned_offset, work_buffer, static_cast(aligned_size))); + if (work_buffer != static_cast(buffer)) { + std::memcpy(buffer, work_buffer + offset - aligned_offset, size); + } + + bool cached = false; + + /* Handle head cache if needed. */ + if (head_cache_needed) { + R_TRY(this->PrepareAllocation()); + + SharedCache cache(this); + while (true) { + R_UNLESS(cache.AcquireFetchableCache(), fs::ResultOutOfResource()); + + UniqueCache fetch_cache(this); + const auto upgrade_result = fetch_cache.Upgrade(cache); + R_TRY(upgrade_result.first); + if (upgrade_result.second) { + R_TRY(fetch_cache.FetchFromBuffer(aligned_offset, work_buffer, static_cast(aligned_size))); + break; + } + } + + cached = true; + } + + /* Handle tail cache if needed. */ + if (tail_cache_needed && (!head_cache_needed || aligned_size > static_cast(m_block_size))) { + if (!cached) { + R_TRY(this->PrepareAllocation()); + } + + SharedCache cache(this); + while (true) { + R_UNLESS(cache.AcquireFetchableCache(), fs::ResultOutOfResource()); + + UniqueCache fetch_cache(this); + const auto upgrade_result = fetch_cache.Upgrade(cache); + R_TRY(upgrade_result.first); + if (upgrade_result.second) { + const s64 tail_cache_offset = util::AlignDown(offset + static_cast(size), m_block_size); + const size_t tail_cache_size = static_cast(aligned_size - tail_cache_offset + aligned_offset); + R_TRY(fetch_cache.FetchFromBuffer(tail_cache_offset, work_buffer + tail_cache_offset - aligned_offset, tail_cache_size)); + break; + } + } + } + + if (cached) { + R_TRY(this->ControlDirtiness()); + } + + return ResultSuccess(); + } + + Result BufferedStorage::WriteCore(s64 offset, const void *buffer, size_t size) { + AMS_ASSERT(m_caches != nullptr); + AMS_ASSERT(buffer != nullptr); + + /* Validate the offset. */ + const auto base_storage_size = m_base_storage_size; + R_UNLESS(offset >= 0, fs::ResultInvalidOffset()); + R_UNLESS(offset <= base_storage_size, fs::ResultInvalidOffset()); + + /* Setup tracking variables. */ + size_t remaining_size = static_cast(std::min(size, base_storage_size - offset)); + s64 cur_offset = offset; + s64 buf_offset = 0; + + /* Repeatedly read until we're done. */ + while (remaining_size > 0) { + /* Determine how much to read this iteration. */ + const auto *cur_src = static_cast(buffer) + buf_offset; + size_t cur_size = 0; + + if (!util::IsAligned(cur_offset, m_block_size)) { + const size_t aligned_size = m_block_size - (cur_offset & (m_block_size - 1)); + cur_size = std::min(aligned_size, remaining_size); + } else if (remaining_size < m_block_size) { + cur_size = remaining_size; + } else { + cur_size = util::AlignDown(remaining_size, m_block_size); + } + + if (cur_size <= m_block_size) { + SharedCache cache(this); + if (!cache.AcquireNextOverlappedCache(cur_offset, cur_size)) { + R_TRY(this->PrepareAllocation()); + while (true) { + R_UNLESS(cache.AcquireFetchableCache(), fs::ResultOutOfResource()); + + UniqueCache fetch_cache(this); + const auto upgrade_result = fetch_cache.Upgrade(cache); + R_TRY(upgrade_result.first); + if (upgrade_result.second) { + R_TRY(fetch_cache.Fetch(cur_offset)); + break; + } + } + } + cache.Write(cur_offset, cur_src, cur_size); + + buffers::EnableBlockingBufferManagerAllocation(); + + R_TRY(this->ControlDirtiness()); + } else { + { + SharedCache cache(this); + while (cache.AcquireNextOverlappedCache(cur_offset, cur_size)) { + R_TRY(cache.Flush()); + cache.Invalidate(); + } + } + + R_TRY(m_base_storage.Write(cur_offset, cur_src, cur_size)); + + buffers::EnableBlockingBufferManagerAllocation(); + } + + remaining_size -= cur_size; + cur_offset += cur_size; + buf_offset += cur_size; + } + + return ResultSuccess(); + } + +} diff --git a/libraries/libstratosphere/source/fssystem/fssystem_hierarchical_integrity_verification_storage.cpp b/libraries/libstratosphere/source/fssystem/fssystem_hierarchical_integrity_verification_storage.cpp new file mode 100644 index 0000000..02ebe77 --- /dev/null +++ b/libraries/libstratosphere/source/fssystem/fssystem_hierarchical_integrity_verification_storage.cpp @@ -0,0 +1,352 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::fssystem { + + namespace { + + constexpr inline u32 IntegrityVerificationStorageMagic = util::FourCC<'I','V','F','C'>::Code; + constexpr inline u32 IntegrityVerificationStorageVersion = 0x00020000; + constexpr inline u32 IntegrityVerificationStorageVersionMask = 0xFFFF0000; + + constexpr inline auto MaxSaveDataFsDataCacheEntryCount = 32; + constexpr inline auto MaxSaveDataFsHashCacheEntryCount = 4; + constexpr inline auto MaxRomFsDataCacheEntryCount = 24; + constexpr inline auto MaxRomFsHashCacheEntryCount = 8; + + constexpr inline auto AccessCountMax = 5; + constexpr inline auto AccessTimeout = TimeSpan::FromMilliSeconds(10); + + os::Semaphore g_read_semaphore(AccessCountMax, AccessCountMax); + os::Semaphore g_write_semaphore(AccessCountMax, AccessCountMax); + + constexpr inline const char MasterKey[] = "HierarchicalIntegrityVerificationStorage::Master"; + constexpr inline const char L1Key[] = "HierarchicalIntegrityVerificationStorage::L1"; + constexpr inline const char L2Key[] = "HierarchicalIntegrityVerificationStorage::L2"; + constexpr inline const char L3Key[] = "HierarchicalIntegrityVerificationStorage::L3"; + constexpr inline const char L4Key[] = "HierarchicalIntegrityVerificationStorage::L4"; + constexpr inline const char L5Key[] = "HierarchicalIntegrityVerificationStorage::L5"; + + constexpr inline const struct { + const char *key; + size_t size; + } KeyArray[] = { + { MasterKey, sizeof(MasterKey) }, + { L1Key, sizeof(L1Key) }, + { L2Key, sizeof(L2Key) }, + { L3Key, sizeof(L3Key) }, + { L4Key, sizeof(L4Key) }, + { L5Key, sizeof(L5Key) }, + }; + + } + + /* Instantiate the global random generation function. */ + constinit HierarchicalIntegrityVerificationStorage::GenerateRandomFunction HierarchicalIntegrityVerificationStorage::s_generate_random = nullptr; + + Result HierarchicalIntegrityVerificationStorageControlArea::QuerySize(HierarchicalIntegrityVerificationSizeSet *out, const InputParam &input_param, s32 layer_count, s64 data_size) { + /* Validate preconditions. */ + AMS_ASSERT(out != nullptr); + AMS_ASSERT((static_cast(IntegrityMinLayerCount) <= layer_count) && (layer_count <= static_cast(IntegrityMaxLayerCount))); + for (s32 level = 0; level < (layer_count - 1); ++level) { + AMS_ASSERT(input_param.level_block_size[level] > 0); + AMS_ASSERT(IsPowerOfTwo(static_cast(input_param.level_block_size[level]))); + } + + /* Set the control size. */ + out->control_size = sizeof(HierarchicalIntegrityVerificationMetaInformation); + + /* Determine the level sizes. */ + s64 level_size[IntegrityMaxLayerCount]; + s32 level = layer_count - 1; + + level_size[level] = util::AlignUp(data_size, input_param.level_block_size[level - 1]); + --level; + + for (/* ... */; level > 0; --level) { + level_size[level] = util::AlignUp(level_size[level + 1] / input_param.level_block_size[level] * HashSize, input_param.level_block_size[level - 1]); + } + + /* Determine the master size. */ + level_size[0] = level_size[1] / input_param.level_block_size[0] * HashSize; + + /* Set the master size. */ + out->master_hash_size = level_size[0]; + + /* Set the level sizes. */ + for (level = 1; level < layer_count - 1; ++level) { + out->layered_hash_sizes[level - 1] = level_size[level]; + } + + return ResultSuccess(); + } + + Result HierarchicalIntegrityVerificationStorageControlArea::Expand(fs::SubStorage meta_storage, const HierarchicalIntegrityVerificationMetaInformation &meta) { + /* Check the meta size. */ + { + s64 meta_size = 0; + R_TRY(meta_storage.GetSize(std::addressof(meta_size))); + R_UNLESS(meta_size >= static_cast(sizeof(meta)), fs::ResultInvalidSize()); + } + + /* Validate both the previous and new metas. */ + { + /* Read the previous meta. */ + HierarchicalIntegrityVerificationMetaInformation prev_meta = {}; + R_TRY(meta_storage.Read(0, std::addressof(prev_meta), sizeof(prev_meta))); + + /* Validate both magics. */ + R_UNLESS(prev_meta.magic == IntegrityVerificationStorageMagic, fs::ResultIncorrectIntegrityVerificationMagic()); + R_UNLESS(prev_meta.magic == meta.magic, fs::ResultIncorrectIntegrityVerificationMagic()); + + /* Validate both versions. */ + R_UNLESS(prev_meta.version == IntegrityVerificationStorageVersion, fs::ResultUnsupportedVersion()); + R_UNLESS(prev_meta.version == meta.version, fs::ResultUnsupportedVersion()); + } + + /* Write the new meta. */ + R_TRY(meta_storage.Write(0, std::addressof(meta), sizeof(meta))); + R_TRY(meta_storage.Flush()); + + return ResultSuccess(); + } + + Result HierarchicalIntegrityVerificationStorageControlArea::Initialize(fs::SubStorage meta_storage) { + /* Check the meta size. */ + { + s64 meta_size = 0; + R_TRY(meta_storage.GetSize(std::addressof(meta_size))); + R_UNLESS(meta_size >= static_cast(sizeof(m_meta)), fs::ResultInvalidSize()); + } + + /* Set the storage and read the meta. */ + m_storage = meta_storage; + R_TRY(m_storage.Read(0, std::addressof(m_meta), sizeof(m_meta))); + + /* Validate the meta magic. */ + R_UNLESS(m_meta.magic == IntegrityVerificationStorageMagic, fs::ResultIncorrectIntegrityVerificationMagic()); + + /* Validate the meta version. */ + R_UNLESS((m_meta.version & IntegrityVerificationStorageVersionMask) == (IntegrityVerificationStorageVersion & IntegrityVerificationStorageVersionMask), fs::ResultUnsupportedVersion()); + + return ResultSuccess(); + } + + void HierarchicalIntegrityVerificationStorageControlArea::Finalize() { + m_storage = fs::SubStorage(); + } + + Result HierarchicalIntegrityVerificationStorage::Initialize(const HierarchicalIntegrityVerificationInformation &info, HierarchicalStorageInformation storage, FileSystemBufferManagerSet *bufs, IHash256GeneratorFactory *hgf, os::SdkRecursiveMutex *mtx, fs::StorageType storage_type) { + /* Validate preconditions. */ + AMS_ASSERT(bufs != nullptr); + AMS_ASSERT(IntegrityMinLayerCount <= info.max_layers && info.max_layers <= IntegrityMaxLayerCount); + + /* Set member variables. */ + m_max_layers = info.max_layers; + m_buffers = bufs; + m_mutex = mtx; + + /* Determine our cache counts. */ + const auto max_data_cache_entry_count = (storage_type == fs::StorageType_SaveData) ? MaxSaveDataFsDataCacheEntryCount : MaxRomFsDataCacheEntryCount; + const auto max_hash_cache_entry_count = (storage_type == fs::StorageType_SaveData) ? MaxSaveDataFsHashCacheEntryCount : MaxRomFsHashCacheEntryCount; + + /* Initialize the top level verification storage. */ + { + fs::HashSalt mac; + crypto::GenerateHmacSha256(mac.value, sizeof(mac), info.seed.value, sizeof(info.seed), KeyArray[0].key, KeyArray[0].size); + m_verify_storages[0].Initialize(storage[HierarchicalStorageInformation::MasterStorage], storage[HierarchicalStorageInformation::Layer1Storage], static_cast(1) << info.info[0].block_order, HashSize, m_buffers->buffers[m_max_layers - 2], hgf, mac, false, storage_type); + } + + /* Ensure we don't leak state if further initialization goes wrong. */ + auto top_verif_guard = SCOPE_GUARD { + m_verify_storages[0].Finalize(); + + m_data_size = -1; + m_buffers = nullptr; + m_mutex = nullptr; + }; + + /* Initialize the top level buffer storage. */ + R_TRY(m_buffer_storages[0].Initialize(m_buffers->buffers[0], m_mutex, std::addressof(m_verify_storages[0]), info.info[0].size, static_cast(1) << info.info[0].block_order, max_hash_cache_entry_count, false, 0x10, false, storage_type)); + auto top_buffer_guard = SCOPE_GUARD { m_buffer_storages[0].Finalize(); }; + + /* Prepare to initialize the level storages. */ + s32 level = 0; + + /* Ensure we don't leak state if further initialization goes wrong. */ + auto level_guard = SCOPE_GUARD { + m_verify_storages[level + 1].Finalize(); + for (/* ... */; level > 0; --level) { + m_buffer_storages[level].Finalize(); + m_verify_storages[level].Finalize(); + } + }; + + /* Initialize the level storages. */ + for (/* ... */; level < m_max_layers - 3; ++level) { + /* Initialize the verification storage. */ + { + fs::SubStorage buffer_storage(std::addressof(m_buffer_storages[level]), 0, info.info[level].size); + fs::HashSalt mac; + crypto::GenerateHmacSha256(mac.value, sizeof(mac), info.seed.value, sizeof(info.seed), KeyArray[level + 1].key, KeyArray[level + 1].size); + m_verify_storages[level + 1].Initialize(buffer_storage, storage[level + 2], static_cast(1) << info.info[level + 1].block_order, static_cast(1) << info.info[level].block_order, m_buffers->buffers[m_max_layers - 2], hgf, mac, false, storage_type); + } + + /* Initialize the buffer storage. */ + R_TRY(m_buffer_storages[level + 1].Initialize(m_buffers->buffers[level + 1], m_mutex, std::addressof(m_verify_storages[level + 1]), info.info[level + 1].size, static_cast(1) << info.info[level + 1].block_order, max_hash_cache_entry_count, false, 0x11 + static_cast(level), false, storage_type)); + } + + /* Initialize the final level storage. */ + { + /* Initialize the verification storage. */ + { + fs::SubStorage buffer_storage(std::addressof(m_buffer_storages[level]), 0, info.info[level].size); + fs::HashSalt mac; + crypto::GenerateHmacSha256(mac.value, sizeof(mac), info.seed.value, sizeof(info.seed), KeyArray[level + 1].key, KeyArray[level + 1].size); + m_verify_storages[level + 1].Initialize(buffer_storage, storage[level + 2], static_cast(1) << info.info[level + 1].block_order, static_cast(1) << info.info[level].block_order, m_buffers->buffers[m_max_layers - 2], hgf, mac, true, storage_type); + } + + /* Initialize the buffer storage. */ + R_TRY(m_buffer_storages[level + 1].Initialize(m_buffers->buffers[level + 1], m_mutex, std::addressof(m_verify_storages[level + 1]), info.info[level + 1].size, static_cast(1) << info.info[level + 1].block_order, max_data_cache_entry_count, true, 0x11 + static_cast(level), true, storage_type)); + } + + /* Set the data size. */ + m_data_size = info.info[level + 1].size; + + /* We succeeded. */ + level_guard.Cancel(); + top_buffer_guard.Cancel(); + top_verif_guard.Cancel(); + return ResultSuccess(); + } + + void HierarchicalIntegrityVerificationStorage::Finalize() { + if (m_data_size >= 0) { + m_data_size = 0; + + m_buffers = nullptr; + m_mutex = nullptr; + + for (s32 level = m_max_layers - 2; level >= 0; --level) { + m_buffer_storages[level].Finalize(); + m_verify_storages[level].Finalize(); + } + + m_data_size = -1; + } + } + + Result HierarchicalIntegrityVerificationStorage::Read(s64 offset, void *buffer, size_t size) { + /* Validate preconditions. */ + AMS_ASSERT(m_data_size >= 0); + + /* Succeed if zero-size. */ + R_SUCCEED_IF(size == 0); + + /* Validate arguments. */ + R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); + + /* Acquire access to the read semaphore. */ + if (!g_read_semaphore.TimedAcquire(AccessTimeout)) { + for (auto level = m_max_layers - 2; level >= 0; --level) { + R_TRY(m_buffer_storages[level].Flush()); + } + g_read_semaphore.Acquire(); + } + + /* Ensure that we release the semaphore when done. */ + ON_SCOPE_EXIT { g_read_semaphore.Release(); }; + + /* Read the data. */ + R_TRY(m_buffer_storages[m_max_layers - 2].Read(offset, buffer, size)); + return ResultSuccess(); + } + + Result HierarchicalIntegrityVerificationStorage::Write(s64 offset, const void *buffer, size_t size) { + /* Validate preconditions. */ + AMS_ASSERT(m_data_size >= 0); + + /* Succeed if zero-size. */ + R_SUCCEED_IF(size == 0); + + /* Validate arguments. */ + R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); + + /* Acquire access to the write semaphore. */ + if (!g_write_semaphore.TimedAcquire(AccessTimeout)) { + for (auto level = m_max_layers - 2; level >= 0; --level) { + R_TRY(m_buffer_storages[level].Flush()); + } + g_write_semaphore.Acquire(); + } + + /* Ensure that we release the semaphore when done. */ + ON_SCOPE_EXIT { g_write_semaphore.Release(); }; + + /* Write the data. */ + R_TRY(m_buffer_storages[m_max_layers - 2].Write(offset, buffer, size)); + m_is_written_for_rollback = true; + return ResultSuccess(); + } + + Result HierarchicalIntegrityVerificationStorage::GetSize(s64 *out) { + AMS_ASSERT(out != nullptr); + AMS_ASSERT(m_data_size >= 0); + *out = m_data_size; + return ResultSuccess(); + } + + Result HierarchicalIntegrityVerificationStorage::Flush() { + return ResultSuccess(); + } + + Result HierarchicalIntegrityVerificationStorage::OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) { + switch (op_id) { + case fs::OperationId::FillZero: + case fs::OperationId::DestroySignature: + { + R_TRY(m_buffer_storages[m_max_layers - 2].OperateRange(dst, dst_size, op_id, offset, size, src, src_size)); + m_is_written_for_rollback = true; + return ResultSuccess(); + } + case fs::OperationId::Invalidate: + case fs::OperationId::QueryRange: + { + R_TRY(m_buffer_storages[m_max_layers - 2].OperateRange(dst, dst_size, op_id, offset, size, src, src_size)); + return ResultSuccess(); + } + default: + return fs::ResultUnsupportedOperationInHierarchicalIntegrityVerificationStorageB(); + } + } + + Result HierarchicalIntegrityVerificationStorage::Commit() { + for (s32 level = m_max_layers - 2; level >= 0; --level) { + R_TRY(m_buffer_storages[level].Commit()); + } + return ResultSuccess(); + } + + Result HierarchicalIntegrityVerificationStorage::OnRollback() { + for (s32 level = m_max_layers - 2; level >= 0; --level) { + R_TRY(m_buffer_storages[level].OnRollback()); + } + m_is_written_for_rollback = false; + return ResultSuccess(); + } + +} diff --git a/libraries/libstratosphere/source/fssystem/fssystem_integrity_romfs_storage.cpp b/libraries/libstratosphere/source/fssystem/fssystem_integrity_romfs_storage.cpp index 4354bf1..63580fb 100644 --- a/libraries/libstratosphere/source/fssystem/fssystem_integrity_romfs_storage.cpp +++ b/libraries/libstratosphere/source/fssystem/fssystem_integrity_romfs_storage.cpp @@ -17,7 +17,7 @@ namespace ams::fssystem { - Result IntegrityRomFsStorage::Initialize(save::HierarchicalIntegrityVerificationInformation level_hash_info, Hash master_hash, save::HierarchicalIntegrityVerificationStorage::HierarchicalStorageInformation storage_info, fs::IBufferManager *bm, IHash256GeneratorFactory *hgf) { + Result IntegrityRomFsStorage::Initialize(HierarchicalIntegrityVerificationInformation level_hash_info, Hash master_hash, HierarchicalIntegrityVerificationStorage::HierarchicalStorageInformation storage_info, fs::IBufferManager *bm, IHash256GeneratorFactory *hgf) { /* Validate preconditions. */ AMS_ASSERT(bm != nullptr); diff --git a/libraries/libstratosphere/source/fssystem/fssystem_integrity_verification_storage.cpp b/libraries/libstratosphere/source/fssystem/fssystem_integrity_verification_storage.cpp new file mode 100644 index 0000000..7618bae --- /dev/null +++ b/libraries/libstratosphere/source/fssystem/fssystem_integrity_verification_storage.cpp @@ -0,0 +1,487 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::fssystem { + + Result IntegrityVerificationStorage::Initialize(fs::SubStorage hs, fs::SubStorage ds, s64 verif_block_size, s64 upper_layer_verif_block_size, fs::IBufferManager *bm, fssystem::IHash256GeneratorFactory *hgf, const fs::HashSalt &salt, bool is_real_data, fs::StorageType storage_type) { + /* Validate preconditions. */ + AMS_ASSERT(verif_block_size >= HashSize); + AMS_ASSERT(bm != nullptr); + AMS_ASSERT(hgf != nullptr); + + /* Set storages. */ + m_hash_storage = hs; + m_data_storage = ds; + + /* Set hash generator factory. */ + m_hash_generator_factory = hgf; + + /* Set verification block sizes. */ + m_verification_block_size = verif_block_size; + m_verification_block_order = ILog2(static_cast(verif_block_size)); + AMS_ASSERT(m_verification_block_size == (1l << m_verification_block_order)); + + /* Set buffer manager. */ + m_buffer_manager = bm; + + /* Set upper layer block sizes. */ + upper_layer_verif_block_size = std::max(upper_layer_verif_block_size, HashSize); + m_upper_layer_verification_block_size = upper_layer_verif_block_size; + m_upper_layer_verification_block_order = ILog2(static_cast(upper_layer_verif_block_size)); + AMS_ASSERT(m_upper_layer_verification_block_size == (1l << m_upper_layer_verification_block_order)); + + /* Validate sizes. */ + { + s64 hash_size = 0; + s64 data_size = 0; + AMS_ASSERT(R_SUCCEEDED(m_hash_storage.GetSize(std::addressof(hash_size)))); + AMS_ASSERT(R_SUCCEEDED(m_data_storage.GetSize(std::addressof(hash_size)))); + AMS_ASSERT(((hash_size / HashSize) * m_verification_block_size) >= data_size); + AMS_UNUSED(hash_size, data_size); + } + + /* Set salt. */ + std::memcpy(m_salt.value, salt.value, fs::HashSalt::Size); + + /* Set data and storage type. */ + m_is_real_data = is_real_data; + m_storage_type = storage_type; + return ResultSuccess(); + } + + void IntegrityVerificationStorage::Finalize() { + if (m_buffer_manager != nullptr) { + m_hash_storage = fs::SubStorage(); + m_data_storage = fs::SubStorage(); + m_buffer_manager = nullptr; + } + } + + Result IntegrityVerificationStorage::Read(s64 offset, void *buffer, size_t size) { + /* Although we support zero-size reads, we expect non-zero sizes. */ + AMS_ASSERT(size != 0); + + /* Validate other preconditions. */ + AMS_ASSERT(util::IsAligned(offset, static_cast(m_verification_block_size))); + AMS_ASSERT(util::IsAligned(size, static_cast(m_verification_block_size))); + + /* Succeed if zero size. */ + R_SUCCEED_IF(size == 0); + + /* Validate arguments. */ + R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); + + /* Validate the offset. */ + s64 data_size; + R_TRY(m_data_storage.GetSize(std::addressof(data_size))); + R_UNLESS(offset <= data_size, fs::ResultInvalidOffset()); + + /* Validate the access range. */ + R_UNLESS(IStorage::CheckAccessRange(offset, size, util::AlignUp(data_size, static_cast(m_verification_block_size))), fs::ResultOutOfRange()); + + /* Determine the read extents. */ + size_t read_size = size; + if (static_cast(offset + read_size) > data_size) { + /* Determine the padding sizes. */ + s64 padding_offset = data_size - offset; + size_t padding_size = static_cast(m_verification_block_size - (padding_offset & (m_verification_block_size - 1))); + AMS_ASSERT(static_cast(padding_size) < m_verification_block_size); + + /* Clear the padding. */ + std::memset(static_cast(buffer) + padding_offset, 0, padding_size); + + /* Set the new in-bounds size. */ + read_size = static_cast(data_size - offset); + } + + /* Perform the read. */ + { + auto clear_guard = SCOPE_GUARD { std::memset(buffer, 0, size); }; + R_TRY(m_data_storage.Read(offset, buffer, read_size)); + clear_guard.Cancel(); + } + + /* Verify the signatures. */ + Result verify_hash_result = ResultSuccess(); + + /* Create hash generator. */ + auto generator = m_hash_generator_factory->Create(); + + /* Prepare to validate the signatures. */ + const auto signature_count = size >> m_verification_block_order; + PooledBuffer signature_buffer(signature_count * sizeof(BlockHash), sizeof(BlockHash)); + const auto buffer_count = std::min(signature_count, signature_buffer.GetSize() / sizeof(BlockHash)); + + size_t verified_count = 0; + while (verified_count < signature_count) { + /* Read the current signatures. */ + const auto cur_count = std::min(buffer_count, signature_count - verified_count); + auto cur_result = this->ReadBlockSignature(signature_buffer.GetBuffer(), signature_buffer.GetSize(), offset + (verified_count << m_verification_block_order), cur_count << m_verification_block_order); + + /* Temporarily increase our priority. */ + ScopedThreadPriorityChanger cp(+1, ScopedThreadPriorityChanger::Mode::Relative); + + /* Loop over each signature we read. */ + for (size_t i = 0; i < cur_count && R_SUCCEEDED(cur_result); ++i) { + const auto verified_size = (verified_count + i) << m_verification_block_order; + u8 *cur_buf = static_cast(buffer) + verified_size; + cur_result = this->VerifyHash(cur_buf, reinterpret_cast(signature_buffer.GetBuffer()) + i, generator); + + /* If the data is corrupted, clear the corrupted parts. */ + if (fs::ResultIntegrityVerificationStorageCorrupted::Includes(cur_result)) { + std::memset(cur_buf, 0, m_verification_block_size); + + /* Set the result if we should. */ + if (!fs::ResultClearedRealDataVerificationFailed::Includes(cur_result) && m_storage_type != fs::StorageType_Authoring) { + verify_hash_result = cur_result; + } + + cur_result = ResultSuccess(); + } + } + + /* If we failed, clear and return. */ + if (R_FAILED(cur_result)) { + std::memset(buffer, 0, size); + return cur_result; + } + + /* Advance. */ + verified_count += cur_count; + } + + return verify_hash_result; + } + + Result IntegrityVerificationStorage::Write(s64 offset, const void *buffer, size_t size) { + /* Succeed if zero size. */ + R_SUCCEED_IF(size == 0); + + /* Validate arguments. */ + R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); + R_UNLESS(IStorage::CheckOffsetAndSize(offset, size), fs::ResultInvalidOffset()); + + /* Validate the offset. */ + s64 data_size; + R_TRY(m_data_storage.GetSize(std::addressof(data_size))); + R_UNLESS(offset < data_size, fs::ResultInvalidOffset()); + + /* Validate the access range. */ + R_UNLESS(IStorage::CheckAccessRange(offset, size, util::AlignUp(data_size, static_cast(m_verification_block_size))), fs::ResultOutOfRange()); + + /* Validate preconditions. */ + AMS_ASSERT(util::IsAligned(offset, m_verification_block_size)); + AMS_ASSERT(util::IsAligned(size, m_verification_block_size)); + AMS_ASSERT(offset <= data_size); + AMS_ASSERT(static_cast(offset + size) < data_size + m_verification_block_size); + + /* Validate that if writing past the end, all extra data is zero padding. */ + if (static_cast(offset + size) > data_size) { + const u8 *padding_cur = static_cast(buffer) + data_size - offset; + const u8 *padding_end = padding_cur + (offset + size - data_size); + + while (padding_cur < padding_end) { + AMS_ASSERT((*padding_cur) == 0); + ++padding_cur; + } + } + + /* Determine the unpadded size to write. */ + auto write_size = size; + if (static_cast(offset + write_size) > data_size) { + write_size = static_cast(data_size - offset); + R_SUCCEED_IF(write_size == 0); + } + + /* Determine the size we're writing in blocks. */ + const auto aligned_write_size = util::AlignUp(write_size, m_verification_block_size); + + /* Write the updated block signatures. */ + Result update_result = ResultSuccess(); + size_t updated_count = 0; + { + const auto signature_count = aligned_write_size >> m_verification_block_order; + PooledBuffer signature_buffer(signature_count * sizeof(BlockHash), sizeof(BlockHash)); + const auto buffer_count = std::min(signature_count, signature_buffer.GetSize() / sizeof(BlockHash)); + + auto generator = m_hash_generator_factory->Create(); + + while (updated_count < signature_count) { + const auto cur_count = std::min(buffer_count, signature_count - updated_count); + + /* Calculate the hash with temporarily increased priority. */ + { + ScopedThreadPriorityChanger cp(+1, ScopedThreadPriorityChanger::Mode::Relative); + + for (size_t i = 0; i < cur_count; ++i) { + const auto updated_size = (updated_count + i) << m_verification_block_order; + this->CalcBlockHash(reinterpret_cast(signature_buffer.GetBuffer()) + i, reinterpret_cast(buffer) + updated_size, generator); + } + } + + /* Write the new block signatures. */ + if (R_FAILED((update_result = this->WriteBlockSignature(signature_buffer.GetBuffer(), signature_buffer.GetSize(), offset + (updated_count << m_verification_block_order), cur_count << m_verification_block_order)))) { + break; + } + + /* Advance. */ + updated_count += cur_count; + } + } + + /* Write the data. */ + R_TRY(m_data_storage.Write(offset, buffer, std::min(write_size, updated_count << m_verification_block_order))); + + return update_result; + } + + Result IntegrityVerificationStorage::GetSize(s64 *out) { + return m_data_storage.GetSize(out); + } + + Result IntegrityVerificationStorage::Flush() { + /* Flush both storages. */ + R_TRY(m_hash_storage.Flush()); + R_TRY(m_data_storage.Flush()); + return ResultSuccess(); + } + + Result IntegrityVerificationStorage::OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) { + /* Validate preconditions. */ + AMS_ASSERT(util::IsAligned(offset, static_cast(m_verification_block_size))); + AMS_ASSERT(util::IsAligned(size, static_cast(m_verification_block_size))); + + switch (op_id) { + case fs::OperationId::FillZero: + { + /* Clear should only be called for save data. */ + AMS_ASSERT(m_storage_type == fs::StorageType_SaveData); + + /* Validate the range. */ + s64 data_size = 0; + R_TRY(m_data_storage.GetSize(std::addressof(data_size))); + R_UNLESS(0 <= offset && offset <= data_size, fs::ResultInvalidOffset()); + + /* Determine the extents to clear. */ + const auto sign_offset = (offset >> m_verification_block_order) * HashSize; + const auto sign_size = (std::min(size, data_size - offset) >> m_verification_block_order) * HashSize; + + /* Allocate a work buffer. */ + const auto buf_size = static_cast(std::min(sign_size, static_cast(1) << (m_upper_layer_verification_block_order + 2))); + std::unique_ptr buf = fs::impl::MakeUnique(buf_size); + R_UNLESS(buf != nullptr, fs::ResultAllocationFailureInIntegrityVerificationStorageA()); + + /* Clear the work buffer. */ + std::memset(buf.get(), 0, buf_size); + + /* Clear in chunks. */ + auto remaining_size = sign_size; + + while (remaining_size > 0) { + const auto cur_size = static_cast(std::min(remaining_size, static_cast(buf_size))); + R_TRY(m_hash_storage.Write(sign_offset + sign_size - remaining_size, buf.get(), cur_size)); + remaining_size -= cur_size; + } + + return ResultSuccess(); + } + case fs::OperationId::DestroySignature: + { + /* Clear Signature should only be called for save data. */ + AMS_ASSERT(m_storage_type == fs::StorageType_SaveData); + + /* Validate the range. */ + s64 data_size = 0; + R_TRY(m_data_storage.GetSize(std::addressof(data_size))); + R_UNLESS(0 <= offset && offset <= data_size, fs::ResultInvalidOffset()); + + /* Determine the extents to clear the signature for. */ + const auto sign_offset = (offset >> m_verification_block_order) * HashSize; + const auto sign_size = (std::min(size, data_size - offset) >> m_verification_block_order) * HashSize; + + /* Allocate a work buffer. */ + std::unique_ptr buf = fs::impl::MakeUnique(sign_size); + R_UNLESS(buf != nullptr, fs::ResultAllocationFailureInIntegrityVerificationStorageB()); + + /* Read the existing signature. */ + R_TRY(m_hash_storage.Read(sign_offset, buf.get(), sign_size)); + + /* Clear the signature. */ + /* This sets all bytes to FF, with the verification bit cleared. */ + for (auto i = 0; i < sign_size; ++i) { + buf[i] ^= ((i + 1) % HashSize == 0 ? 0x7F : 0xFF); + } + + /* Write the cleared signature. */ + return m_hash_storage.Write(sign_offset, buf.get(), sign_size); + } + case fs::OperationId::Invalidate: + { + /* Only allow cache invalidation for RomFs. */ + R_UNLESS(m_storage_type != fs::StorageType_SaveData, fs::ResultUnsupportedOperationInIntegrityVerificationStorageB()); + + + /* Operate on our storages. */ + R_TRY(m_hash_storage.OperateRange(dst, dst_size, op_id, 0, std::numeric_limits::max(), src, src_size)); + R_TRY(m_data_storage.OperateRange(dst, dst_size, op_id, offset, size, src, src_size)); + + return ResultSuccess(); + } + case fs::OperationId::QueryRange: + { + /* Validate the range. */ + s64 data_size = 0; + R_TRY(m_data_storage.GetSize(std::addressof(data_size))); + R_UNLESS(0 <= offset && offset <= data_size, fs::ResultInvalidOffset()); + + /* Determine the real size to query. */ + const auto actual_size = std::min(size, data_size - offset); + + /* Query the data storage. */ + R_TRY(m_data_storage.OperateRange(dst, dst_size, op_id, offset, actual_size, src, src_size)); + + return ResultSuccess(); + } + default: + return fs::ResultUnsupportedOperationInIntegrityVerificationStorageC(); + } + } + + void IntegrityVerificationStorage::CalcBlockHash(BlockHash *out, const void *buffer, size_t block_size, std::unique_ptr &generator) const { + /* Initialize the generator. */ + generator->Initialize(); + + /* If calculating for save data, hash the salt. */ + if (m_storage_type == fs::StorageType_SaveData) { + generator->Update(m_salt.value, sizeof(m_salt)); + } + + /* Update with the buffer and get the hash. */ + generator->Update(buffer, block_size); + generator->GetHash(out, sizeof(*out)); + + /* Set the validation bit, if the hash is for save data. */ + if (m_storage_type == fs::StorageType_SaveData) { + SetValidationBit(out); + } + } + + Result IntegrityVerificationStorage::ReadBlockSignature(void *dst, size_t dst_size, s64 offset, size_t size) { + /* Validate preconditions. */ + AMS_ASSERT(dst != nullptr); + AMS_ASSERT(util::IsAligned(offset, static_cast(m_verification_block_size))); + AMS_ASSERT(util::IsAligned(size, static_cast(m_verification_block_size))); + + /* Determine where to read the signature. */ + const s64 sign_offset = (offset >> m_verification_block_order) * HashSize; + const auto sign_size = static_cast((size >> m_verification_block_order) * HashSize); + AMS_ASSERT(dst_size >= sign_size); + AMS_UNUSED(dst_size); + + /* Create a guard in the event of failure. */ + auto clear_guard = SCOPE_GUARD { std::memset(dst, 0, sign_size); }; + + /* Validate that we can read the signature. */ + s64 hash_size; + R_TRY(m_hash_storage.GetSize(std::addressof(hash_size))); + const bool range_valid = static_cast(sign_offset + sign_size) <= hash_size; + AMS_ASSERT(range_valid); + R_UNLESS(range_valid, fs::ResultOutOfRange()); + + /* Read the signature. */ + R_TRY(m_hash_storage.Read(sign_offset, dst, sign_size)); + + /* We succeeded. */ + clear_guard.Cancel(); + return ResultSuccess(); + } + + Result IntegrityVerificationStorage::WriteBlockSignature(const void *src, size_t src_size, s64 offset, size_t size) { + /* Validate preconditions. */ + AMS_ASSERT(src != nullptr); + AMS_ASSERT(util::IsAligned(offset, static_cast(m_verification_block_size))); + + /* Determine where to write the signature. */ + const s64 sign_offset = (offset >> m_verification_block_order) * HashSize; + const auto sign_size = static_cast((size >> m_verification_block_order) * HashSize); + AMS_ASSERT(src_size >= sign_size); + AMS_UNUSED(src_size); + + /* Write the signature. */ + R_TRY(m_hash_storage.Write(sign_offset, src, sign_size)); + + /* We succeeded. */ + return ResultSuccess(); + } + + Result IntegrityVerificationStorage::VerifyHash(const void *buf, BlockHash *hash, std::unique_ptr &generator) { + /* Validate preconditions. */ + AMS_ASSERT(buf != nullptr); + AMS_ASSERT(hash != nullptr); + + /* Get the comparison hash. */ + auto &cmp_hash = *hash; + + /* If save data, check if the data is uninitialized. */ + if (m_storage_type == fs::StorageType_SaveData) { + bool is_cleared = false; + R_TRY(this->IsCleared(std::addressof(is_cleared), cmp_hash)); + R_UNLESS(!is_cleared, fs::ResultClearedRealDataVerificationFailed()); + } + + /* Get the calculated hash. */ + BlockHash calc_hash; + this->CalcBlockHash(std::addressof(calc_hash), buf, generator); + + /* Check that the signatures are equal. */ + if (!crypto::IsSameBytes(std::addressof(cmp_hash), std::addressof(calc_hash), sizeof(BlockHash))) { + /* Clear the comparison hash. */ + std::memset(std::addressof(cmp_hash), 0, sizeof(cmp_hash)); + + /* Return the appropriate result. */ + if (m_is_real_data) { + return fs::ResultUnclearedRealDataVerificationFailed(); + } else { + return fs::ResultNonRealDataVerificationFailed(); + } + } + + return ResultSuccess(); + } + + Result IntegrityVerificationStorage::IsCleared(bool *is_cleared, const BlockHash &hash) { + /* Validate preconditions. */ + AMS_ASSERT(is_cleared != nullptr); + AMS_ASSERT(m_storage_type == fs::StorageType_SaveData); + + /* Default to uncleared. */ + *is_cleared = false; + + /* Succeed if the validation bit is set. */ + R_SUCCEED_IF(IsValidationBit(std::addressof(hash))); + + /* Otherwise, we expect the hash to be all zero. */ + for (size_t i = 0; i < sizeof(hash.hash); ++i) { + R_UNLESS(hash.hash[i] == 0, fs::ResultInvalidZeroHash()); + } + + /* Set cleared. */ + *is_cleared = true; + return ResultSuccess(); + } + +} diff --git a/libraries/libstratosphere/source/fssystem/fssystem_nca_file_system_driver.cpp b/libraries/libstratosphere/source/fssystem/fssystem_nca_file_system_driver.cpp index 36305c8..d1f9c46 100644 --- a/libraries/libstratosphere/source/fssystem/fssystem_nca_file_system_driver.cpp +++ b/libraries/libstratosphere/source/fssystem/fssystem_nca_file_system_driver.cpp @@ -549,7 +549,7 @@ R_TRY(base_storage->GetSize(std::addressof(base_size))); /* Create buffered storage. */ - auto buffered_storage = fssystem::AllocateShared(); + auto buffered_storage = fssystem::AllocateShared(); R_UNLESS(buffered_storage != nullptr, fs::ResultAllocationFailureInAllocateShared()); /* Initialize the buffered storage. */ @@ -655,7 +655,7 @@ R_TRY(this->CreateAesCtrStorage(std::addressof(decrypted_storage), std::move(enc_storage), offset + meta_offset, sparse_info.MakeAesCtrUpperIv(upper_iv), AlignmentStorageRequirement_None)); /* Create meta storage. */ - auto meta_storage = fssystem::AllocateShared(); + auto meta_storage = fssystem::AllocateShared(); R_UNLESS(meta_storage != nullptr, fs::ResultAllocationFailureInAllocateShared()); /* Initialize the meta storage. */ @@ -789,7 +789,7 @@ R_TRY(this->CreateAesCtrStorage(std::addressof(decrypted_storage), std::move(enc_storage), offset + meta_offset, upper_iv, AlignmentStorageRequirement_None)); /* Create meta storage. */ - auto meta_storage = fssystem::AllocateShared(); + auto meta_storage = fssystem::AllocateShared(); R_UNLESS(meta_storage != nullptr, fs::ResultAllocationFailureInAllocateShared()); /* Initialize the meta storage. */ @@ -921,7 +921,7 @@ R_UNLESS(patch_info.indirect_offset + patch_info.indirect_size <= base_size, fs::ResultNcaBaseStorageOutOfRangeE()); /* Allocate the meta storage. */ - auto meta_storage = fssystem::AllocateShared(); + auto meta_storage = fssystem::AllocateShared(); R_UNLESS(meta_storage != nullptr, fs::ResultAllocationFailureInAllocateShared()); /* Initialize the meta storage. */ @@ -954,7 +954,7 @@ AMS_ASSERT(util::IsAligned(indirect_data_size, NcaHeader::XtsBlockSize)); /* Create the indirect data storage. */ - auto indirect_data_storage = fssystem::AllocateShared(); + auto indirect_data_storage = fssystem::AllocateShared(); R_UNLESS(indirect_data_storage != nullptr, fs::ResultAllocationFailureInAllocateShared()); /* Initialize the indirect data storage. */ @@ -1061,15 +1061,15 @@ AMS_ASSERT(base_storage != nullptr); /* Define storage types. */ - using VerificationStorage = save::HierarchicalIntegrityVerificationStorage; + using VerificationStorage = HierarchicalIntegrityVerificationStorage; using StorageInfo = VerificationStorage::HierarchicalStorageInformation; /* Validate the meta info. */ - save::HierarchicalIntegrityVerificationInformation level_hash_info; + HierarchicalIntegrityVerificationInformation level_hash_info; std::memcpy(std::addressof(level_hash_info), std::addressof(meta_info.level_hash_info), sizeof(level_hash_info)); - R_UNLESS(save::IntegrityMinLayerCount <= level_hash_info.max_layers, fs::ResultInvalidHierarchicalIntegrityVerificationLayerCount()); - R_UNLESS(level_hash_info.max_layers <= save::IntegrityMaxLayerCount, fs::ResultInvalidHierarchicalIntegrityVerificationLayerCount()); + R_UNLESS(IntegrityMinLayerCount <= level_hash_info.max_layers, fs::ResultInvalidHierarchicalIntegrityVerificationLayerCount()); + R_UNLESS(level_hash_info.max_layers <= IntegrityMaxLayerCount, fs::ResultInvalidHierarchicalIntegrityVerificationLayerCount()); /* Get the base storage size. */ s64 base_storage_size; diff --git a/libraries/libstratosphere/source/fssystem/save/fssystem_block_cache_buffered_storage.cpp b/libraries/libstratosphere/source/fssystem/save/fssystem_block_cache_buffered_storage.cpp deleted file mode 100644 index bb79b54..0000000 --- a/libraries/libstratosphere/source/fssystem/save/fssystem_block_cache_buffered_storage.cpp +++ /dev/null @@ -1,1072 +0,0 @@ -/* - * Copyright (c) Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#include - -namespace ams::fssystem::save { - - BlockCacheBufferedStorage::BlockCacheBufferedStorage() : m_mutex(), m_data_storage(), m_last_result(ResultSuccess()), m_data_size(), m_verification_block_size(), m_verification_block_shift(), m_flags(), m_buffer_level(-1), m_block_cache_manager() - { - /* ... */ - } - - BlockCacheBufferedStorage::~BlockCacheBufferedStorage() { - this->Finalize(); - } - - Result BlockCacheBufferedStorage::Initialize(fs::IBufferManager *bm, os::SdkRecursiveMutex *mtx, IStorage *data, s64 data_size, size_t verif_block_size, s32 max_cache_entries, bool is_real_data, s8 buffer_level, bool is_keep_burst_mode, fs::StorageType storage_type) { - /* Validate preconditions. */ - AMS_ASSERT(data != nullptr); - AMS_ASSERT(bm != nullptr); - AMS_ASSERT(mtx != nullptr); - AMS_ASSERT(m_mutex == nullptr); - AMS_ASSERT(m_data_storage == nullptr); - AMS_ASSERT(max_cache_entries > 0); - - /* Initialize our manager. */ - R_TRY(m_block_cache_manager.Initialize(bm, max_cache_entries)); - - /* Set members. */ - m_mutex = mtx; - m_data_storage = data; - m_data_size = data_size; - m_verification_block_size = verif_block_size; - m_last_result = ResultSuccess(); - m_flags = 0; - m_buffer_level = buffer_level; - m_storage_type = storage_type; - - /* Calculate block shift. */ - m_verification_block_shift = ILog2(static_cast(verif_block_size)); - AMS_ASSERT(static_cast(UINT64_C(1) << m_verification_block_shift) == m_verification_block_size); - - /* Set burst mode. */ - this->SetKeepBurstMode(is_keep_burst_mode); - - /* Set real data cache. */ - this->SetRealDataCache(is_real_data); - - R_SUCCEED(); - } - - void BlockCacheBufferedStorage::Finalize() { - if (m_block_cache_manager.IsInitialized()) { - /* Invalidate all cache entries. */ - this->InvalidateAllCacheEntries(); - - /* Finalize our block cache manager. */ - m_block_cache_manager.Finalize(); - - /* Clear members. */ - m_mutex = nullptr; - m_data_storage = nullptr; - m_data_size = 0; - m_verification_block_size = 0; - m_verification_block_shift = 0; - } - } - - Result BlockCacheBufferedStorage::Read(s64 offset, void *buffer, size_t size) { - /* Validate pre-conditions. */ - AMS_ASSERT(m_data_storage != nullptr); - AMS_ASSERT(m_block_cache_manager.IsInitialized()); - - /* Ensure we aren't already in a failed state. */ - R_TRY(m_last_result); - - /* Succeed if zero-size. */ - R_SUCCEED_IF(size == 0); - - /* Validate arguments. */ - R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); - - /* Determine the extents to read. */ - s64 read_offset = offset; - size_t read_size = size; - - R_UNLESS(read_offset < m_data_size, fs::ResultInvalidOffset()); - - if (static_cast(read_offset + read_size) > m_data_size) { - read_size = static_cast(m_data_size - read_offset); - } - - /* Determine the aligned range to read. */ - const size_t block_alignment = m_verification_block_size; - s64 aligned_offset = util::AlignDown(read_offset, block_alignment); - s64 aligned_offset_end = util::AlignUp(read_offset + read_size, block_alignment); - - AMS_ASSERT(0 <= aligned_offset && aligned_offset_end <= static_cast(util::AlignUp(m_data_size, block_alignment))); - - /* Try to read using cache. */ - char *dst = static_cast(buffer); - { - /* Determine if we can do bulk reads. */ - constexpr s64 BulkReadSizeMax = 2_MB; - const bool bulk_read_enabled = (read_offset != aligned_offset || static_cast(read_offset + read_size) != aligned_offset_end) && aligned_offset_end - aligned_offset <= BulkReadSizeMax; - - /* Read the head cache. */ - CacheEntry head_entry = {}; - MemoryRange head_range = {}; - bool head_cache_needed = true; - R_TRY(this->ReadHeadCache(std::addressof(head_range), std::addressof(head_entry), std::addressof(head_cache_needed), std::addressof(read_offset), std::addressof(aligned_offset), aligned_offset_end, std::addressof(dst), std::addressof(read_size))); - - /* We may be done after reading the head cache, so check if we are. */ - R_SUCCEED_IF(aligned_offset >= aligned_offset_end); - - /* Ensure we destroy the head buffer. */ - auto head_guard = SCOPE_GUARD { m_block_cache_manager.ReleaseCacheEntry(std::addressof(head_entry), head_range); }; - - /* Read the tail cache. */ - CacheEntry tail_entry = {}; - MemoryRange tail_range = {}; - bool tail_cache_needed = true; - R_TRY(this->ReadTailCache(std::addressof(tail_range), std::addressof(tail_entry), std::addressof(tail_cache_needed), read_offset, aligned_offset, std::addressof(aligned_offset_end), dst, std::addressof(read_size))); - - /* We may be done after reading the tail cache, so check if we are. */ - R_SUCCEED_IF(aligned_offset >= aligned_offset_end); - - /* Ensure that we destroy the tail buffer. */ - auto tail_guard = SCOPE_GUARD { m_block_cache_manager.ReleaseCacheEntry(std::addressof(tail_entry), tail_range); }; - - /* Try to do a bulk read. */ - if (bulk_read_enabled) { - /* The bulk read will destroy our head/tail buffers. */ - head_guard.Cancel(); - tail_guard.Cancel(); - - do { - /* Do the bulk read. If we fail due to pooled buffer allocation failing, fall back to the normal read codepath. */ - R_TRY_CATCH(this->BulkRead(read_offset, dst, read_size, std::addressof(head_range), std::addressof(tail_range), std::addressof(head_entry), std::addressof(tail_entry), head_cache_needed, tail_cache_needed)) { - R_CATCH(fs::ResultAllocationFailurePooledBufferNotEnoughSize) { break; } - } R_END_TRY_CATCH; - - /* Se successfully did a bulk read, so we're done. */ - R_SUCCEED(); - } while (0); - } - } - - /* Read the data using non-bulk reads. */ - while (aligned_offset < aligned_offset_end) { - /* Ensure that there is data for us to read. */ - AMS_ASSERT(read_size > 0); - - /* If conditions allow us to, read in burst mode. This doesn't use the cache. */ - if (this->IsEnabledKeepBurstMode() && read_offset == aligned_offset && (block_alignment * 2 <= read_size)) { - const size_t aligned_size = util::AlignDown(read_size, block_alignment); - - /* Flush the entries. */ - R_TRY(this->UpdateLastResult(this->FlushRangeCacheEntries(read_offset, aligned_size, false))); - - /* Read the data. */ - R_TRY(this->UpdateLastResult(m_data_storage->Read(read_offset, dst, aligned_size))); - - /* Advance. */ - dst += aligned_size; - read_offset += aligned_size; - read_size -= aligned_size; - aligned_offset += aligned_size; - } else { - /* Get the buffer associated with what we're reading. */ - CacheEntry entry; - MemoryRange range; - R_TRY(this->UpdateLastResult(this->GetAssociateBuffer(std::addressof(range), std::addressof(entry), aligned_offset, static_cast(aligned_offset_end - aligned_offset), true))); - - /* Determine where to read data into, and ensure that our entry is aligned. */ - char *src = reinterpret_cast(range.first); - AMS_ASSERT(util::IsAligned(entry.range.size, block_alignment)); - - /* If the entry isn't cached, read the data. */ - if (!entry.is_cached) { - if (const Result result = m_data_storage->Read(entry.range.offset, src, entry.range.size); R_FAILED(result)) { - m_block_cache_manager.ReleaseCacheEntry(std::addressof(entry), range); - return this->UpdateLastResult(result); - } - entry.is_cached = true; - } - - /* Validate the entry extents. */ - AMS_ASSERT(static_cast(entry.range.offset) <= aligned_offset); - AMS_ASSERT(aligned_offset < entry.range.GetEndOffset()); - AMS_ASSERT(aligned_offset <= read_offset); - - /* Copy the data. */ - { - /* Determine where and how much to copy. */ - const s64 buffer_offset = read_offset - entry.range.offset; - const size_t copy_size = std::min(read_size, static_cast(entry.range.GetEndOffset() - read_offset)); - - /* Actually copy the data. */ - std::memcpy(dst, src + buffer_offset, copy_size); - - /* Advance. */ - dst += copy_size; - read_offset += copy_size; - read_size -= copy_size; - } - - /* Release the cache entry. */ - R_TRY(this->UpdateLastResult(this->StoreOrDestroyBuffer(range, std::addressof(entry)))); - aligned_offset = entry.range.GetEndOffset(); - } - } - - /* Ensure that we read all the data. */ - AMS_ASSERT(read_size == 0); - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::Write(s64 offset, const void *buffer, size_t size) { - /* Validate pre-conditions. */ - AMS_ASSERT(m_data_storage != nullptr); - AMS_ASSERT(m_block_cache_manager.IsInitialized()); - - /* Ensure we aren't already in a failed state. */ - R_TRY(m_last_result); - - /* Succeed if zero-size. */ - R_SUCCEED_IF(size == 0); - - /* Validate arguments. */ - R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); - - /* Determine the extents to read. */ - R_UNLESS(offset < m_data_size, fs::ResultInvalidOffset()); - - if (static_cast(offset + size) > m_data_size) { - size = static_cast(m_data_size - offset); - } - - /* The actual extents may be zero-size, so succeed if that's the case. */ - R_SUCCEED_IF(size == 0); - - /* Determine the aligned range to read. */ - const size_t block_alignment = m_verification_block_size; - s64 aligned_offset = util::AlignDown(offset, block_alignment); - const s64 aligned_offset_end = util::AlignUp(offset + size, block_alignment); - - AMS_ASSERT(0 <= aligned_offset && aligned_offset_end <= static_cast(util::AlignUp(m_data_size, block_alignment))); - - /* Write the data. */ - const u8 *src = static_cast(buffer); - while (aligned_offset < aligned_offset_end) { - /* If conditions allow us to, write in burst mode. This doesn't use the cache. */ - if (this->IsEnabledKeepBurstMode() && offset == aligned_offset && (block_alignment * 2 <= size)) { - const size_t aligned_size = util::AlignDown(size, block_alignment); - - /* Flush the entries. */ - R_TRY(this->UpdateLastResult(this->FlushRangeCacheEntries(offset, aligned_size, true))); - - /* Read the data. */ - R_TRY(this->UpdateLastResult(m_data_storage->Write(offset, src, aligned_size))); - - /* Set blocking buffer manager allocations. */ - buffers::EnableBlockingBufferManagerAllocation(); - - /* Advance. */ - src += aligned_size; - offset += aligned_size; - size -= aligned_size; - aligned_offset += aligned_size; - } else { - /* Get the buffer associated with what we're writing. */ - CacheEntry entry; - MemoryRange range; - R_TRY(this->UpdateLastResult(this->GetAssociateBuffer(std::addressof(range), std::addressof(entry), aligned_offset, static_cast(aligned_offset_end - aligned_offset), true))); - - /* Determine where to write data into. */ - char *dst = reinterpret_cast(range.first); - - /* If the entry isn't cached and we're writing a partial entry, read in the entry. */ - if (!entry.is_cached && ((offset != entry.range.offset) || (offset + size < static_cast(entry.range.GetEndOffset())))) { - if (Result result = m_data_storage->Read(entry.range.offset, dst, entry.range.size); R_FAILED(result)) { - m_block_cache_manager.ReleaseCacheEntry(std::addressof(entry), range); - return this->UpdateLastResult(result); - } - } - entry.is_cached = true; - - /* Validate the entry extents. */ - AMS_ASSERT(static_cast(entry.range.offset) <= aligned_offset); - AMS_ASSERT(aligned_offset < entry.range.GetEndOffset()); - AMS_ASSERT(aligned_offset <= offset); - - /* Copy the data. */ - { - /* Determine where and how much to copy. */ - const s64 buffer_offset = offset - entry.range.offset; - const size_t copy_size = std::min(size, static_cast(entry.range.GetEndOffset() - offset)); - - /* Actually copy the data. */ - std::memcpy(dst + buffer_offset, src, copy_size); - - /* Advance. */ - src += copy_size; - offset += copy_size; - size -= copy_size; - } - - /* Set the entry as write-back. */ - entry.is_write_back = true; - - /* Set blocking buffer manager allocations. */ - buffers::EnableBlockingBufferManagerAllocation(); - - /* Store the associated buffer. */ - CacheIndex index; - R_TRY(this->UpdateLastResult(this->StoreOrDestroyBuffer(std::addressof(index), range, std::addressof(entry)))); - - /* Set the after aligned offset. */ - aligned_offset = entry.range.GetEndOffset(); - - /* If we need to, flush the cache entry. */ - if (index >= 0 && IsEnabledKeepBurstMode() && offset == aligned_offset && (block_alignment * 2 <= size)) { - R_TRY(this->UpdateLastResult(this->FlushCacheEntry(index, false))); - } - - } - } - - /* Ensure that didn't end up in a failure state. */ - R_TRY(m_last_result); - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::GetSize(s64 *out) { - /* Validate pre-conditions. */ - AMS_ASSERT(out != nullptr); - AMS_ASSERT(m_data_storage != nullptr); - - /* Set the size. */ - *out = m_data_size; - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::Flush() { - /* Validate pre-conditions. */ - AMS_ASSERT(m_data_storage != nullptr); - AMS_ASSERT(m_block_cache_manager.IsInitialized()); - - /* Ensure we aren't already in a failed state. */ - R_TRY(m_last_result); - - /* Flush all cache entries. */ - R_TRY(this->UpdateLastResult(this->FlushAllCacheEntries())); - - /* Flush the data storage. */ - R_TRY(this->UpdateLastResult(m_data_storage->Flush())); - - /* Set blocking buffer manager allocations. */ - buffers::EnableBlockingBufferManagerAllocation(); - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) { - AMS_UNUSED(src, src_size); - - /* Validate pre-conditions. */ - AMS_ASSERT(m_data_storage != nullptr); - - switch (op_id) { - case fs::OperationId::FillZero: - { - R_TRY(this->FillZeroImpl(offset, size)); - R_SUCCEED(); - } - case fs::OperationId::DestroySignature: - { - R_TRY(this->DestroySignatureImpl(offset, size)); - R_SUCCEED(); - } - case fs::OperationId::Invalidate: - { - R_UNLESS(m_storage_type != fs::StorageType_SaveData, fs::ResultUnsupportedOperationInBlockCacheBufferedStorageB()); - R_TRY(this->InvalidateImpl()); - R_SUCCEED(); - } - case fs::OperationId::QueryRange: - { - R_TRY(this->QueryRangeImpl(dst, dst_size, offset, size)); - R_SUCCEED(); - } - default: - R_THROW(fs::ResultUnsupportedOperationInBlockCacheBufferedStorageC()); - } - } - - Result BlockCacheBufferedStorage::Commit() { - /* Validate pre-conditions. */ - AMS_ASSERT(m_data_storage != nullptr); - AMS_ASSERT(m_block_cache_manager.IsInitialized()); - - /* Ensure we aren't already in a failed state. */ - R_TRY(m_last_result); - - /* Flush all cache entries. */ - R_TRY(this->UpdateLastResult(this->FlushAllCacheEntries())); - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::OnRollback() { - /* Validate pre-conditions. */ - AMS_ASSERT(m_block_cache_manager.IsInitialized()); - - /* Ensure we aren't already in a failed state. */ - R_TRY(m_last_result); - - /* Release all valid entries back to the buffer manager. */ - const auto max_cache_entry_count = m_block_cache_manager.GetCount(); - for (auto index = 0; index < max_cache_entry_count; index++) { - if (const auto &entry = m_block_cache_manager[index]; entry.is_valid) { - m_block_cache_manager.InvalidateCacheEntry(index); - } - } - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::FillZeroImpl(s64 offset, s64 size) { - /* Ensure we aren't already in a failed state. */ - R_TRY(m_last_result); - - /* Get our storage size. */ - s64 storage_size = 0; - R_TRY(this->UpdateLastResult(m_data_storage->GetSize(std::addressof(storage_size)))); - - /* Check the access range. */ - R_UNLESS(0 <= offset && offset < storage_size, fs::ResultInvalidOffset()); - - /* Determine the extents to data signature for. */ - auto start_offset = util::AlignDown(offset, m_verification_block_size); - auto end_offset = util::AlignUp(std::min(offset + size, storage_size), m_verification_block_size); - - /* Flush the entries. */ - R_TRY(this->UpdateLastResult(this->FlushRangeCacheEntries(offset, size, true))); - - /* Handle any data before or after the aligned range. */ - if (start_offset < offset || offset + size < end_offset) { - /* Allocate a work buffer. */ - std::unique_ptr work = fs::impl::MakeUnique(m_verification_block_size); - R_UNLESS(work != nullptr, fs::ResultAllocationFailureInBlockCacheBufferedStorageB()); - - /* Handle data before the aligned range. */ - if (start_offset < offset) { - /* Read the block. */ - R_TRY(this->UpdateLastResult(m_data_storage->Read(start_offset, work.get(), m_verification_block_size))); - - /* Determine the partial extents to clear. */ - const auto clear_offset = static_cast(offset - start_offset); - const auto clear_size = static_cast(std::min(static_cast(m_verification_block_size - clear_offset), size)); - - /* Clear the partial block. */ - std::memset(work.get() + clear_offset, 0, clear_size); - - /* Write the partially cleared block. */ - R_TRY(this->UpdateLastResult(m_data_storage->Write(start_offset, work.get(), m_verification_block_size))); - - /* Update the start offset. */ - start_offset += m_verification_block_size; - - /* Set blocking buffer manager allocations. */ - buffers::EnableBlockingBufferManagerAllocation(); - } - - /* Handle data after the aligned range. */ - if (start_offset < offset + size && offset + size < end_offset) { - /* Read the block. */ - const auto last_offset = end_offset - m_verification_block_size; - R_TRY(this->UpdateLastResult(m_data_storage->Read(last_offset, work.get(), m_verification_block_size))); - - /* Clear the partial block. */ - const auto clear_size = static_cast((offset + size) - last_offset); - std::memset(work.get(), 0, clear_size); - - /* Write the partially cleared block. */ - R_TRY(this->UpdateLastResult(m_data_storage->Write(last_offset, work.get(), m_verification_block_size))); - - /* Update the end offset. */ - end_offset -= m_verification_block_size; - - /* Set blocking buffer manager allocations. */ - buffers::EnableBlockingBufferManagerAllocation(); - } - } - - /* We're done if there's no data to clear. */ - R_SUCCEED_IF(start_offset == end_offset); - - /* Clear the signature for the aligned range. */ - R_TRY(this->UpdateLastResult(m_data_storage->OperateRange(fs::OperationId::FillZero, start_offset, end_offset - start_offset))); - - /* Set blocking buffer manager allocations. */ - buffers::EnableBlockingBufferManagerAllocation(); - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::DestroySignatureImpl(s64 offset, s64 size) { - /* Ensure we aren't already in a failed state. */ - R_TRY(m_last_result); - - /* Get our storage size. */ - s64 storage_size = 0; - R_TRY(this->UpdateLastResult(m_data_storage->GetSize(std::addressof(storage_size)))); - - /* Check the access range. */ - R_UNLESS(0 <= offset && offset < storage_size, fs::ResultInvalidOffset()); - - /* Determine the extents to clear signature for. */ - const auto start_offset = util::AlignUp(offset, m_verification_block_size); - const auto end_offset = util::AlignDown(std::min(offset + size, storage_size), m_verification_block_size); - - /* Flush the entries. */ - R_TRY(this->UpdateLastResult(this->FlushRangeCacheEntries(offset, size, true))); - - /* Clear the signature for the aligned range. */ - R_TRY(this->UpdateLastResult(m_data_storage->OperateRange(fs::OperationId::DestroySignature, start_offset, end_offset - start_offset))); - - /* Set blocking buffer manager allocations. */ - buffers::EnableBlockingBufferManagerAllocation(); - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::InvalidateImpl() { - /* Invalidate cache entries. */ - { - std::scoped_lock lk(*m_mutex); - - m_block_cache_manager.Invalidate(); - } - - /* Invalidate the aligned range. */ - { - Result result = m_data_storage->OperateRange(fs::OperationId::Invalidate, 0, std::numeric_limits::max()); - AMS_ASSERT(!fs::ResultBufferAllocationFailed::Includes(result)); - R_TRY(result); - } - - /* Clear our last result if we should. */ - if (fs::ResultIntegrityVerificationStorageCorrupted::Includes(m_last_result)) { - m_last_result = ResultSuccess(); - } - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::QueryRangeImpl(void *dst, size_t dst_size, s64 offset, s64 size) { - /* Get our storage size. */ - s64 storage_size = 0; - R_TRY(this->GetSize(std::addressof(storage_size))); - - /* Determine the extents we can actually query. */ - const auto actual_size = std::min(size, storage_size - offset); - const auto aligned_offset = util::AlignDown(offset, m_verification_block_size); - const auto aligned_offset_end = util::AlignUp(offset + actual_size, m_verification_block_size); - const auto aligned_size = aligned_offset_end - aligned_offset; - - /* Query the aligned range. */ - R_TRY(this->UpdateLastResult(m_data_storage->OperateRange(dst, dst_size, fs::OperationId::QueryRange, aligned_offset, aligned_size, nullptr, 0))); - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::GetAssociateBuffer(MemoryRange *out_range, CacheEntry *out_entry, s64 offset, size_t ideal_size, bool is_allocate_for_write) { - AMS_UNUSED(is_allocate_for_write); - - /* Validate pre-conditions. */ - AMS_ASSERT(m_data_storage != nullptr); - AMS_ASSERT(m_block_cache_manager.IsInitialized()); - AMS_ASSERT(out_range != nullptr); - AMS_ASSERT(out_entry != nullptr); - - /* Lock our mutex. */ - std::scoped_lock lk(*m_mutex); - - /* Get the maximum cache entry count. */ - const CacheIndex max_cache_entry_count = m_block_cache_manager.GetCount(); - - /* Locate the index of the cache entry, if present. */ - CacheIndex index; - size_t actual_size = ideal_size; - for (index = 0; index < max_cache_entry_count; ++index) { - if (const auto &entry = m_block_cache_manager[index]; entry.IsAllocated()) { - if (entry.range.IsIncluded(offset)) { - break; - } - - if (offset <= entry.range.offset && entry.range.offset < static_cast(offset + actual_size)) { - actual_size = static_cast(entry.range.offset - offset); - } - } - } - - /* Clear the out range. */ - out_range->first = 0; - out_range->second = 0; - - /* If we located an entry, use it. */ - if (index != max_cache_entry_count) { - m_block_cache_manager.AcquireCacheEntry(out_entry, out_range, index); - - actual_size = out_entry->range.size - (offset - out_entry->range.offset); - } - - /* If we don't have an out entry, allocate one. */ - if (out_range->first == 0) { - /* Ensure that the allocatable size is above a threshold. */ - const auto size_threshold = m_block_cache_manager.GetAllocator()->GetTotalSize() / 8; - if (m_block_cache_manager.GetAllocator()->GetTotalAllocatableSize() < size_threshold) { - R_TRY(this->FlushAllCacheEntries()); - } - - /* Decide in advance on a block alignment. */ - const size_t block_alignment = m_verification_block_size; - - /* Ensure that the size we request is valid. */ - { - AMS_ASSERT(actual_size >= 1); - actual_size = std::min(actual_size, block_alignment * 2); - } - AMS_ASSERT(actual_size >= block_alignment); - - /* Allocate a buffer. */ - R_TRY(buffers::AllocateBufferUsingBufferManagerContext(out_range, m_block_cache_manager.GetAllocator(), actual_size, fs::IBufferManager::BufferAttribute(m_buffer_level), [=](const MemoryRange &buffer) { - return buffer.first != 0 && block_alignment <= buffer.second; - }, AMS_CURRENT_FUNCTION_NAME)); - - /* Ensure our size is accurate. */ - actual_size = std::min(actual_size, out_range->second); - - /* Set the output entry. */ - out_entry->is_valid = true; - out_entry->is_write_back = false; - out_entry->is_cached = false; - out_entry->is_flushing = false; - out_entry->handle = 0; - out_entry->memory_address = 0; - out_entry->memory_size = 0; - out_entry->range.offset = offset; - out_entry->range.size = actual_size; - out_entry->lru_counter = 0; - } - - /* Check that we ended up with a coherent out range. */ - AMS_ASSERT(out_range->second >= out_entry->range.size); - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::StoreOrDestroyBuffer(CacheIndex *out, const MemoryRange &range, CacheEntry *entry) { - /* Validate pre-conditions. */ - AMS_ASSERT(out != nullptr); - - /* Lock our mutex. */ - std::scoped_lock lk(*m_mutex); - - /* In the event that we fail, release our buffer. */ - ON_RESULT_FAILURE { m_block_cache_manager.ReleaseCacheEntry(entry, range); }; - - /* If the entry is write-back, ensure we don't exceed certain dirtiness thresholds. */ - if (entry->is_write_back) { - R_TRY(this->ControlDirtiness()); - } - - /* Get unused cache entry index. */ - CacheIndex empty_index, lru_index; - m_block_cache_manager.GetEmptyCacheEntryIndex(std::addressof(empty_index), std::addressof(lru_index)); - - /* If all entries are valid, we need to invalidate one. */ - if (empty_index == BlockCacheManager::InvalidCacheIndex) { - /* Invalidate the lease recently used entry. */ - empty_index = lru_index; - - /* Get the entry to invalidate, sanity check that we can invalidate it. */ - const CacheEntry &entry_to_invalidate = m_block_cache_manager[empty_index]; - AMS_ASSERT(entry_to_invalidate.is_valid); - AMS_ASSERT(!entry_to_invalidate.is_flushing); - AMS_UNUSED(entry_to_invalidate); - - /* Invalidate the entry. */ - R_TRY(this->FlushCacheEntry(empty_index, true)); - - /* Check that the entry was invalidated successfully. */ - AMS_ASSERT(!entry_to_invalidate.is_valid); - AMS_ASSERT(!entry_to_invalidate.is_flushing); - } - - /* Store the entry. */ - if (m_block_cache_manager.SetCacheEntry(empty_index, *entry, range, fs::IBufferManager::BufferAttribute(m_buffer_level))) { - *out = empty_index; - } else { - *out = BlockCacheManager::InvalidCacheIndex; - } - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::FlushCacheEntry(CacheIndex index, bool invalidate) { - /* Lock our mutex. */ - std::scoped_lock lk(*m_mutex); - - /* Get the entry, sanity check that the entry's state allows for flush. */ - auto &entry = m_block_cache_manager[index]; - AMS_ASSERT(entry.is_valid); - AMS_ASSERT(!entry.is_flushing); - - /* If we're not write back (i.e. an invalidate is happening), just release the buffer. */ - if (!entry.is_write_back) { - AMS_ASSERT(invalidate); - - m_block_cache_manager.InvalidateCacheEntry(index); - - R_SUCCEED(); - } - - /* Note that we've started flushing, while we process. */ - m_block_cache_manager.SetFlushing(index, true); - ON_SCOPE_EXIT { m_block_cache_manager.SetFlushing(index, false); }; - - /* Create and check our memory range. */ - MemoryRange memory_range = fs::IBufferManager::MakeMemoryRange(entry.memory_address, entry.memory_size); - AMS_ASSERT(memory_range.first != 0); - AMS_ASSERT(memory_range.second >= entry.range.size); - - /* Validate the entry's offset. */ - AMS_ASSERT(entry.range.offset >= 0); - AMS_ASSERT(entry.range.offset < m_data_size); - AMS_ASSERT(util::IsAligned(entry.range.offset, m_verification_block_size)); - - /* Write back the data. */ - Result result = ResultSuccess(); - size_t write_size = entry.range.size; - if (R_SUCCEEDED(m_last_result)) { - /* Set blocking buffer manager allocations. */ - result = m_data_storage->Write(entry.range.offset, reinterpret_cast(memory_range.first), write_size); - - /* Check the result. */ - AMS_ASSERT(!fs::ResultBufferAllocationFailed::Includes(result)); - } else { - result = m_last_result; - } - - /* Set that we're not write-back. */ - m_block_cache_manager.SetWriteBack(index, false); - - /* If we're invalidating, release the buffer. Otherwise, register the flushed data. */ - if (invalidate) { - m_block_cache_manager.ReleaseCacheEntry(index, memory_range); - } else { - AMS_ASSERT(entry.is_valid); - m_block_cache_manager.RegisterCacheEntry(index, memory_range, fs::IBufferManager::BufferAttribute(m_buffer_level)); - } - - /* Try to succeed. */ - R_TRY(result); - - /* We succeeded. */ - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::FlushRangeCacheEntries(s64 offset, s64 size, bool invalidate) { - /* Validate pre-conditions. */ - AMS_ASSERT(m_data_storage != nullptr); - AMS_ASSERT(m_block_cache_manager.IsInitialized()); - - /* Iterate over all entries that fall within the range. */ - Result result = ResultSuccess(); - const auto max_cache_entry_count = m_block_cache_manager.GetCount(); - for (auto i = 0; i < max_cache_entry_count; ++i) { - auto &entry = m_block_cache_manager[i]; - if (entry.is_valid && (entry.is_write_back || invalidate) && (entry.range.offset < (offset + size)) && (offset < entry.range.GetEndOffset())) { - const auto cur_result = this->FlushCacheEntry(i, invalidate); - if (R_FAILED(cur_result) && R_SUCCEEDED(result)) { - result = cur_result; - } - } - } - - /* Try to succeed. */ - R_TRY(result); - - /* We succeeded. */ - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::FlushAllCacheEntries() { - R_TRY(this->FlushRangeCacheEntries(0, std::numeric_limits::max(), false)); - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::InvalidateAllCacheEntries() { - R_TRY(this->FlushRangeCacheEntries(0, std::numeric_limits::max(), true)); - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::ControlDirtiness() { - /* Get and validate the max cache entry count. */ - const auto max_cache_entry_count = m_block_cache_manager.GetCount(); - AMS_ASSERT(max_cache_entry_count > 0); - - /* Get size metrics from the buffer manager. */ - const auto total_size = m_block_cache_manager.GetAllocator()->GetTotalSize(); - const auto allocatable_size = m_block_cache_manager.GetAllocator()->GetTotalAllocatableSize(); - - /* If we have enough allocatable space, we don't need to do anything. */ - R_SUCCEED_IF(allocatable_size >= total_size / 4); - - /* Iterate over all entries (up to the threshold) and flush the least recently used dirty entry. */ - constexpr auto Threshold = 2; - for (int n = 0; n < Threshold; ++n) { - auto flushed_index = BlockCacheManager::InvalidCacheIndex; - for (auto index = 0; index < max_cache_entry_count; ++index) { - if (auto &entry = m_block_cache_manager[index]; entry.is_valid && entry.is_write_back) { - if (flushed_index == BlockCacheManager::InvalidCacheIndex || m_block_cache_manager[flushed_index].lru_counter < entry.lru_counter) { - flushed_index = index; - } - } - } - - /* If we can't flush anything, break. */ - if (flushed_index == BlockCacheManager::InvalidCacheIndex) { - break; - } - - R_TRY(this->FlushCacheEntry(flushed_index, false)); - } - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::UpdateLastResult(Result result) { - /* Update the last result. */ - if (R_FAILED(result) && !fs::ResultBufferAllocationFailed::Includes(result) && R_SUCCEEDED(m_last_result)) { - m_last_result = result; - } - - /* Try to succeed with the result. */ - R_TRY(result); - - /* We succeeded. */ - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::ReadHeadCache(MemoryRange *out_range, CacheEntry *out_entry, bool *out_cache_needed, s64 *offset, s64 *aligned_offset, s64 aligned_offset_end, char **buffer, size_t *size) { - /* Valdiate pre-conditions. */ - AMS_ASSERT(out_range != nullptr); - AMS_ASSERT(out_entry != nullptr); - AMS_ASSERT(out_cache_needed != nullptr); - AMS_ASSERT(offset != nullptr); - AMS_ASSERT(aligned_offset != nullptr); - AMS_ASSERT(buffer != nullptr); - AMS_ASSERT(*buffer != nullptr); - AMS_ASSERT(size != nullptr); - - AMS_ASSERT(*aligned_offset < aligned_offset_end); - - /* Iterate over the region. */ - CacheEntry entry = {}; - MemoryRange memory_range = {}; - *out_cache_needed = true; - - while (*aligned_offset < aligned_offset_end) { - /* Get the associated buffer for the offset. */ - R_TRY(this->UpdateLastResult(this->GetAssociateBuffer(std::addressof(memory_range), std::addressof(entry), *aligned_offset, m_verification_block_size, true))); - - /* If the entry isn't cached, we're done. */ - if (!entry.is_cached) { - break; - } - - /* Set cache not needed. */ - *out_cache_needed = false; - - /* Determine the size to copy. */ - const s64 buffer_offset = *offset - entry.range.offset; - const size_t copy_size = std::min(*size, static_cast(entry.range.GetEndOffset() - *offset)); - - /* Copy data from the entry. */ - std::memcpy(*buffer, reinterpret_cast(memory_range.first + buffer_offset), copy_size); - - /* Advance. */ - *buffer += copy_size; - *offset += copy_size; - *size -= copy_size; - *aligned_offset = entry.range.GetEndOffset(); - - /* Handle the buffer. */ - R_TRY(this->UpdateLastResult(this->StoreOrDestroyBuffer(memory_range, std::addressof(entry)))); - } - - /* Set the output entry. */ - *out_entry = entry; - *out_range = memory_range; - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::ReadTailCache(MemoryRange *out_range, CacheEntry *out_entry, bool *out_cache_needed, s64 offset, s64 aligned_offset, s64 *aligned_offset_end, char *buffer, size_t *size) { - /* Valdiate pre-conditions. */ - AMS_ASSERT(out_range != nullptr); - AMS_ASSERT(out_entry != nullptr); - AMS_ASSERT(out_cache_needed != nullptr); - AMS_ASSERT(aligned_offset_end != nullptr); - AMS_ASSERT(buffer != nullptr); - AMS_ASSERT(size != nullptr); - - AMS_ASSERT(aligned_offset < *aligned_offset_end); - - /* Iterate over the region. */ - CacheEntry entry = {}; - MemoryRange memory_range = {}; - *out_cache_needed = true; - - while (aligned_offset < *aligned_offset_end) { - /* Get the associated buffer for the offset. */ - R_TRY(this->UpdateLastResult(this->GetAssociateBuffer(std::addressof(memory_range), std::addressof(entry), *aligned_offset_end - m_verification_block_size, m_verification_block_size, true))); - - /* If the entry isn't cached, we're done. */ - if (!entry.is_cached) { - break; - } - - /* Set cache not needed. */ - *out_cache_needed = false; - - /* Determine the size to copy. */ - const s64 buffer_offset = std::max(static_cast(0), offset - entry.range.offset); - const size_t copy_size = std::min(*size, static_cast(offset + *size - entry.range.offset)); - - /* Copy data from the entry. */ - std::memcpy(buffer + *size - copy_size, reinterpret_cast(memory_range.first + buffer_offset), copy_size); - - /* Advance. */ - *size -= copy_size; - *aligned_offset_end = entry.range.offset; - - /* Handle the buffer. */ - R_TRY(this->UpdateLastResult(this->StoreOrDestroyBuffer(memory_range, std::addressof(entry)))); - } - - /* Set the output entry. */ - *out_entry = entry; - *out_range = memory_range; - - R_SUCCEED(); - } - - Result BlockCacheBufferedStorage::BulkRead(s64 offset, void *buffer, size_t size, MemoryRange *range_head, MemoryRange *range_tail, CacheEntry *entry_head, CacheEntry *entry_tail, bool head_cache_needed, bool tail_cache_needed) { - /* Validate pre-conditions. */ - AMS_ASSERT(buffer != nullptr); - AMS_ASSERT(range_head != nullptr); - AMS_ASSERT(range_tail != nullptr); - AMS_ASSERT(entry_head != nullptr); - AMS_ASSERT(entry_tail != nullptr); - - /* Determine bulk read offsets. */ - const s64 read_offset = offset; - const size_t read_size = size; - const s64 aligned_offset = util::AlignDown(read_offset, m_verification_block_size); - const s64 aligned_offset_end = util::AlignUp(read_offset + read_size, m_verification_block_size); - char *dst = static_cast(buffer); - - /* Prepare to do our reads. */ - auto head_guard = SCOPE_GUARD { m_block_cache_manager.ReleaseCacheEntry(entry_head, *range_head); }; - auto tail_guard = SCOPE_GUARD { m_block_cache_manager.ReleaseCacheEntry(entry_tail, *range_tail); }; - - /* Flush the entries. */ - R_TRY(this->UpdateLastResult(this->FlushRangeCacheEntries(aligned_offset, aligned_offset_end - aligned_offset, false))); - - /* Determine the buffer to read into. */ - PooledBuffer pooled_buffer; - const size_t buffer_size = static_cast(aligned_offset_end - aligned_offset); - char *read_buffer = nullptr; - if (read_offset == aligned_offset && read_size == buffer_size) { - read_buffer = dst; - } else if (tail_cache_needed && entry_tail->range.offset == aligned_offset && entry_tail->range.size == buffer_size) { - read_buffer = reinterpret_cast(range_tail->first); - } else if (head_cache_needed && entry_head->range.offset == aligned_offset && entry_head->range.size == buffer_size) { - read_buffer = reinterpret_cast(range_head->first); - } else { - pooled_buffer.AllocateParticularlyLarge(buffer_size, 1); - R_UNLESS(pooled_buffer.GetSize() >= buffer_size, fs::ResultAllocationFailurePooledBufferNotEnoughSize()); - read_buffer = pooled_buffer.GetBuffer(); - } - - /* Read the data. */ - R_TRY(m_data_storage->Read(aligned_offset, read_buffer, buffer_size)); - - /* Copy the data out. */ - if (dst != read_buffer) { - std::memcpy(dst, read_buffer + read_offset - aligned_offset, read_size); - } - - /* Create a helper to populate our caches. */ - const auto PopulateCacheFromPooledBuffer = [&](CacheEntry *entry, MemoryRange *range) { - AMS_ASSERT(entry != nullptr); - AMS_ASSERT(range != nullptr); - - if (aligned_offset <= entry->range.offset && entry->range.GetEndOffset() <= static_cast(aligned_offset + buffer_size)) { - AMS_ASSERT(!entry->is_cached); - if (reinterpret_cast(range->first) != read_buffer) { - std::memcpy(reinterpret_cast(range->first), read_buffer + entry->range.offset - aligned_offset, entry->range.size); - } - entry->is_cached = true; - } - }; - - /* Populate tail cache if needed. */ - if (tail_cache_needed) { - PopulateCacheFromPooledBuffer(entry_tail, range_tail); - } - - /* Populate head cache if needed. */ - if (head_cache_needed) { - PopulateCacheFromPooledBuffer(entry_head, range_head); - } - - /* If both entries are cached, one may contain the other; in that case, we need only the larger entry. */ - if (entry_head->is_cached && entry_tail->is_cached) { - if (entry_tail->range.offset <= entry_head->range.offset && entry_head->range.GetEndOffset() <= entry_tail->range.GetEndOffset()) { - entry_head->is_cached = false; - } else if (entry_head->range.offset <= entry_tail->range.offset && entry_tail->range.GetEndOffset() <= entry_head->range.GetEndOffset()) { - entry_tail->is_cached = false; - } - } - - /* Destroy the tail cache. */ - tail_guard.Cancel(); - if (entry_tail->is_cached) { - R_TRY(this->UpdateLastResult(this->StoreOrDestroyBuffer(*range_tail, entry_tail))); - } else { - m_block_cache_manager.ReleaseCacheEntry(entry_tail, *range_tail); - } - - /* Destroy the head cache. */ - head_guard.Cancel(); - if (entry_head->is_cached) { - R_TRY(this->UpdateLastResult(this->StoreOrDestroyBuffer(*range_head, entry_head))); - } else { - m_block_cache_manager.ReleaseCacheEntry(entry_head, *range_head); - } - - R_SUCCEED(); - } - -} diff --git a/libraries/libstratosphere/source/fssystem/save/fssystem_buffered_storage.cpp b/libraries/libstratosphere/source/fssystem/save/fssystem_buffered_storage.cpp deleted file mode 100644 index 57f5153..0000000 --- a/libraries/libstratosphere/source/fssystem/save/fssystem_buffered_storage.cpp +++ /dev/null @@ -1,1085 +0,0 @@ -/* - * Copyright (c) Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#include - -namespace ams::fssystem::save { - - namespace { - - constexpr inline uintptr_t InvalidAddress = 0; - constexpr inline s64 InvalidOffset = std::numeric_limits::max(); - - } - - class BufferedStorage::Cache : public ::ams::fs::impl::Newable { - private: - struct FetchParameter { - s64 offset; - void *buffer; - size_t size; - }; - static_assert(util::is_pod::value); - private: - BufferedStorage *m_buffered_storage; - std::pair m_memory_range; - fs::IBufferManager::CacheHandle m_cache_handle; - s64 m_offset; - std::atomic m_is_valid; - std::atomic m_is_dirty; - u8 m_reserved[2]; - s32 m_reference_count; - Cache *m_next; - Cache *m_prev; - public: - Cache() : m_buffered_storage(nullptr), m_memory_range(InvalidAddress, 0), m_cache_handle(), m_offset(InvalidOffset), m_is_valid(false), m_is_dirty(false), m_reference_count(1), m_next(nullptr), m_prev(nullptr) { - /* ... */ - } - - ~Cache() { - this->Finalize(); - } - - void Initialize(BufferedStorage *bs) { - AMS_ASSERT(bs != nullptr); - AMS_ASSERT(m_buffered_storage == nullptr); - - m_buffered_storage = bs; - this->Link(); - } - - void Finalize() { - AMS_ASSERT(m_buffered_storage != nullptr); - AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); - AMS_ASSERT(m_reference_count == 0); - - /* If we're valid, acquire our cache handle and free our buffer. */ - if (this->IsValid()) { - const auto buffer_manager = m_buffered_storage->m_buffer_manager; - if (!m_is_dirty) { - AMS_ASSERT(m_memory_range.first == InvalidAddress); - m_memory_range = buffer_manager->AcquireCache(m_cache_handle); - } - if (m_memory_range.first != InvalidAddress) { - buffer_manager->DeallocateBuffer(m_memory_range.first, m_memory_range.second); - m_memory_range.first = InvalidAddress; - m_memory_range.second = 0; - } - } - - /* Clear all our members. */ - m_buffered_storage = nullptr; - m_offset = InvalidOffset; - m_is_valid = false; - m_is_dirty = false; - m_next = nullptr; - m_prev = nullptr; - } - - void Link() { - AMS_ASSERT(m_buffered_storage != nullptr); - AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); - AMS_ASSERT(m_reference_count > 0); - - if ((--m_reference_count) == 0) { - AMS_ASSERT(m_next == nullptr); - AMS_ASSERT(m_prev == nullptr); - - if (m_buffered_storage->m_next_fetch_cache == nullptr) { - m_buffered_storage->m_next_fetch_cache = this; - m_next = this; - m_prev = this; - } else { - /* Check against a cache being registered twice. */ - { - auto cache = m_buffered_storage->m_next_fetch_cache; - do { - if (cache->IsValid() && this->Hits(cache->m_offset, m_buffered_storage->m_block_size)) { - m_is_valid = false; - break; - } - cache = cache->m_next; - } while (cache != m_buffered_storage->m_next_fetch_cache); - } - - /* Link into the fetch list. */ - { - AMS_ASSERT(m_buffered_storage->m_next_fetch_cache->m_prev != nullptr); - AMS_ASSERT(m_buffered_storage->m_next_fetch_cache->m_prev->m_next == m_buffered_storage->m_next_fetch_cache); - m_next = m_buffered_storage->m_next_fetch_cache; - m_prev = m_buffered_storage->m_next_fetch_cache->m_prev; - m_next->m_prev = this; - m_prev->m_next = this; - } - - /* Insert invalid caches at the start of the list. */ - if (!this->IsValid()) { - m_buffered_storage->m_next_fetch_cache = this; - } - } - - /* If we're not valid, clear our offset. */ - if (!this->IsValid()) { - m_offset = InvalidOffset; - m_is_dirty = false; - } - - /* Ensure our buffer state is coherent. */ - if (m_memory_range.first != InvalidAddress && !m_is_dirty) { - if (this->IsValid()) { - m_cache_handle = m_buffered_storage->m_buffer_manager->RegisterCache(m_memory_range.first, m_memory_range.second, fs::IBufferManager::BufferAttribute()); - } else { - m_buffered_storage->m_buffer_manager->DeallocateBuffer(m_memory_range.first, m_memory_range.second); - } - m_memory_range.first = InvalidAddress; - m_memory_range.second = 0; - } - } - } - - void Unlink() { - AMS_ASSERT(m_buffered_storage != nullptr); - AMS_ASSERT(m_reference_count >= 0); - - if ((++m_reference_count) == 1) { - AMS_ASSERT(m_next != nullptr); - AMS_ASSERT(m_prev != nullptr); - AMS_ASSERT(m_next->m_prev == this); - AMS_ASSERT(m_prev->m_next == this); - - if (m_buffered_storage->m_next_fetch_cache == this) { - if (m_next != this) { - m_buffered_storage->m_next_fetch_cache = m_next; - } else { - m_buffered_storage->m_next_fetch_cache = nullptr; - } - } - - m_buffered_storage->m_next_acquire_cache = this; - - m_next->m_prev = m_prev; - m_prev->m_next = m_next; - m_next = nullptr; - m_prev = nullptr; - } else { - AMS_ASSERT(m_next == nullptr); - AMS_ASSERT(m_prev == nullptr); - } - } - - void Read(s64 offset, void *buffer, size_t size) const { - AMS_ASSERT(m_buffered_storage != nullptr); - AMS_ASSERT(m_next == nullptr); - AMS_ASSERT(m_prev == nullptr); - AMS_ASSERT(this->IsValid()); - AMS_ASSERT(this->Hits(offset, 1)); - AMS_ASSERT(m_memory_range.first != InvalidAddress); - - const auto read_offset = offset - m_offset; - const auto readable_offset_max = m_buffered_storage->m_block_size - size; - const auto cache_buffer = reinterpret_cast(m_memory_range.first) + read_offset; - AMS_ASSERT(read_offset >= 0); - AMS_ASSERT(static_cast(read_offset) <= readable_offset_max); - AMS_UNUSED(readable_offset_max); - - std::memcpy(buffer, cache_buffer, size); - } - - void Write(s64 offset, const void *buffer, size_t size) { - AMS_ASSERT(m_buffered_storage != nullptr); - AMS_ASSERT(m_next == nullptr); - AMS_ASSERT(m_prev == nullptr); - AMS_ASSERT(this->IsValid()); - AMS_ASSERT(this->Hits(offset, 1)); - AMS_ASSERT(m_memory_range.first != InvalidAddress); - - const auto write_offset = offset - m_offset; - const auto writable_offset_max = m_buffered_storage->m_block_size - size; - const auto cache_buffer = reinterpret_cast(m_memory_range.first) + write_offset; - AMS_ASSERT(write_offset >= 0); - AMS_ASSERT(static_cast(write_offset) <= writable_offset_max); - AMS_UNUSED(writable_offset_max); - - std::memcpy(cache_buffer, buffer, size); - m_is_dirty = true; - } - - Result Flush() { - AMS_ASSERT(m_buffered_storage != nullptr); - AMS_ASSERT(m_next == nullptr); - AMS_ASSERT(m_prev == nullptr); - AMS_ASSERT(this->IsValid()); - - if (m_is_dirty) { - AMS_ASSERT(m_memory_range.first != InvalidAddress); - - const auto base_size = m_buffered_storage->m_base_storage_size; - const auto block_size = static_cast(m_buffered_storage->m_block_size); - const auto flush_size = static_cast(std::min(block_size, base_size - m_offset)); - - auto &base_storage = m_buffered_storage->m_base_storage; - const auto cache_buffer = reinterpret_cast(m_memory_range.first); - - R_TRY(base_storage.Write(m_offset, cache_buffer, flush_size)); - m_is_dirty = false; - - buffers::EnableBlockingBufferManagerAllocation(); - } - - return ResultSuccess(); - } - - const std::pair PrepareFetch() { - AMS_ASSERT(m_buffered_storage != nullptr); - AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); - AMS_ASSERT(m_next == nullptr); - AMS_ASSERT(m_prev == nullptr); - AMS_ASSERT(this->IsValid()); - AMS_ASSERT(m_buffered_storage->m_mutex.IsLockedByCurrentThread()); - - std::pair result(ResultSuccess(), false); - if (m_reference_count == 1) { - result.first = this->Flush(); - if (R_SUCCEEDED(result.first)) { - m_is_valid = false; - m_reference_count = 0; - result.second = true; - } - } - - return result; - } - - void UnprepareFetch() { - AMS_ASSERT(m_buffered_storage != nullptr); - AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); - AMS_ASSERT(m_next == nullptr); - AMS_ASSERT(m_prev == nullptr); - AMS_ASSERT(!this->IsValid()); - AMS_ASSERT(!m_is_dirty); - AMS_ASSERT(m_buffered_storage->m_mutex.IsLockedByCurrentThread()); - - m_is_valid = true; - m_reference_count = 1; - } - - Result Fetch(s64 offset) { - AMS_ASSERT(m_buffered_storage != nullptr); - AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); - AMS_ASSERT(m_next == nullptr); - AMS_ASSERT(m_prev == nullptr); - AMS_ASSERT(!this->IsValid()); - AMS_ASSERT(!m_is_dirty); - - if (m_memory_range.first == InvalidAddress) { - R_TRY(this->AllocateFetchBuffer()); - } - - FetchParameter fetch_param = {}; - this->CalcFetchParameter(std::addressof(fetch_param), offset); - - auto &base_storage = m_buffered_storage->m_base_storage; - R_TRY(base_storage.Read(fetch_param.offset, fetch_param.buffer, fetch_param.size)); - m_offset = fetch_param.offset; - AMS_ASSERT(this->Hits(offset, 1)); - - return ResultSuccess(); - } - - Result FetchFromBuffer(s64 offset, const void *buffer, size_t buffer_size) { - AMS_ASSERT(m_buffered_storage != nullptr); - AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); - AMS_ASSERT(m_next == nullptr); - AMS_ASSERT(m_prev == nullptr); - AMS_ASSERT(!this->IsValid()); - AMS_ASSERT(!m_is_dirty); - AMS_ASSERT(util::IsAligned(offset, m_buffered_storage->m_block_size)); - - if (m_memory_range.first == InvalidAddress) { - R_TRY(this->AllocateFetchBuffer()); - } - - FetchParameter fetch_param = {}; - this->CalcFetchParameter(std::addressof(fetch_param), offset); - AMS_ASSERT(fetch_param.offset == offset); - AMS_ASSERT(fetch_param.size <= buffer_size); - AMS_UNUSED(buffer_size); - - std::memcpy(fetch_param.buffer, buffer, fetch_param.size); - m_offset = fetch_param.offset; - AMS_ASSERT(this->Hits(offset, 1)); - - return ResultSuccess(); - } - - bool TryAcquireCache() { - AMS_ASSERT(m_buffered_storage != nullptr); - AMS_ASSERT(m_buffered_storage->m_buffer_manager != nullptr); - AMS_ASSERT(this->IsValid()); - - if (m_memory_range.first != InvalidAddress) { - return true; - } else { - m_memory_range = m_buffered_storage->m_buffer_manager->AcquireCache(m_cache_handle); - m_is_valid = m_memory_range.first != InvalidAddress; - return m_is_valid; - } - } - - void Invalidate() { - AMS_ASSERT(m_buffered_storage != nullptr); - m_is_valid = false; - } - - bool IsValid() const { - AMS_ASSERT(m_buffered_storage != nullptr); - return m_is_valid || m_reference_count > 0; - } - - bool IsDirty() const { - AMS_ASSERT(m_buffered_storage != nullptr); - return m_is_dirty; - } - - bool Hits(s64 offset, s64 size) const { - AMS_ASSERT(m_buffered_storage != nullptr); - const auto block_size = static_cast(m_buffered_storage->m_block_size); - return (offset < m_offset + block_size) && (m_offset < offset + size); - } - private: - Result AllocateFetchBuffer() { - fs::IBufferManager *buffer_manager = m_buffered_storage->m_buffer_manager; - AMS_ASSERT(buffer_manager->AcquireCache(m_cache_handle).first == InvalidAddress); - - auto range_guard = SCOPE_GUARD { m_memory_range.first = InvalidAddress; }; - R_TRY(buffers::AllocateBufferUsingBufferManagerContext(std::addressof(m_memory_range), buffer_manager, m_buffered_storage->m_block_size, fs::IBufferManager::BufferAttribute(), [](const std::pair &buffer) { - return buffer.first != 0; - }, AMS_CURRENT_FUNCTION_NAME)); - - range_guard.Cancel(); - return ResultSuccess(); - } - - void CalcFetchParameter(FetchParameter *out, s64 offset) const { - AMS_ASSERT(out != nullptr); - - const auto block_size = static_cast(m_buffered_storage->m_block_size); - const auto cache_offset = util::AlignDown(offset, m_buffered_storage->m_block_size); - const auto base_size = m_buffered_storage->m_base_storage_size; - const auto cache_size = static_cast(std::min(block_size, base_size - cache_offset)); - const auto cache_buffer = reinterpret_cast(m_memory_range.first); - AMS_ASSERT(offset >= 0); - AMS_ASSERT(offset < base_size); - - out->offset = cache_offset; - out->buffer = cache_buffer; - out->size = cache_size; - } - }; - - class BufferedStorage::SharedCache { - NON_COPYABLE(SharedCache); - NON_MOVEABLE(SharedCache); - friend class UniqueCache; - private: - Cache *m_cache; - Cache *m_start_cache; - BufferedStorage *m_buffered_storage; - public: - explicit SharedCache(BufferedStorage *bs) : m_cache(nullptr), m_start_cache(bs->m_next_acquire_cache), m_buffered_storage(bs) { - AMS_ASSERT(m_buffered_storage != nullptr); - } - - ~SharedCache() { - std::scoped_lock lk(m_buffered_storage->m_mutex); - this->Release(); - } - - bool AcquireNextOverlappedCache(s64 offset, s64 size) { - AMS_ASSERT(m_buffered_storage != nullptr); - - auto is_first = m_cache == nullptr; - const auto start = is_first ? m_start_cache : m_cache + 1; - - AMS_ASSERT(start >= m_buffered_storage->m_caches.get()); - AMS_ASSERT(start <= m_buffered_storage->m_caches.get() + m_buffered_storage->m_cache_count); - - std::scoped_lock lk(m_buffered_storage->m_mutex); - - this->Release(); - AMS_ASSERT(m_cache == nullptr); - - for (auto cache = start; true; ++cache) { - if (m_buffered_storage->m_caches.get() + m_buffered_storage->m_cache_count <= cache) { - cache = m_buffered_storage->m_caches.get(); - } - if (!is_first && cache == m_start_cache) { - break; - } - if (cache->IsValid() && cache->Hits(offset, size) && cache->TryAcquireCache()) { - cache->Unlink(); - m_cache = cache; - return true; - } - is_first = false; - } - - m_cache = nullptr; - return false; - } - - bool AcquireNextDirtyCache() { - AMS_ASSERT(m_buffered_storage != nullptr); - const auto start = m_cache != nullptr ? m_cache + 1 : m_buffered_storage->m_caches.get(); - const auto end = m_buffered_storage->m_caches.get() + m_buffered_storage->m_cache_count; - - AMS_ASSERT(start >= m_buffered_storage->m_caches.get()); - AMS_ASSERT(start <= end); - - this->Release(); - AMS_ASSERT(m_cache == nullptr); - - for (auto cache = start; cache < end; ++cache) { - if (cache->IsValid() && cache->IsDirty() && cache->TryAcquireCache()) { - cache->Unlink(); - m_cache = cache; - return true; - } - } - - m_cache = nullptr; - return false; - } - - bool AcquireNextValidCache() { - AMS_ASSERT(m_buffered_storage != nullptr); - const auto start = m_cache != nullptr ? m_cache + 1 : m_buffered_storage->m_caches.get(); - const auto end = m_buffered_storage->m_caches.get() + m_buffered_storage->m_cache_count; - - AMS_ASSERT(start >= m_buffered_storage->m_caches.get()); - AMS_ASSERT(start <= end); - - this->Release(); - AMS_ASSERT(m_cache == nullptr); - - for (auto cache = start; cache < end; ++cache) { - if (cache->IsValid() && cache->TryAcquireCache()) { - cache->Unlink(); - m_cache = cache; - return true; - } - } - - m_cache = nullptr; - return false; - } - - bool AcquireFetchableCache() { - AMS_ASSERT(m_buffered_storage != nullptr); - - std::scoped_lock lk(m_buffered_storage->m_mutex); - - this->Release(); - AMS_ASSERT(m_cache == nullptr); - - m_cache = m_buffered_storage->m_next_fetch_cache; - if (m_cache != nullptr) { - if (m_cache->IsValid()) { - m_cache->TryAcquireCache(); - } - m_cache->Unlink(); - } - - return m_cache != nullptr; - } - - void Read(s64 offset, void *buffer, size_t size) { - AMS_ASSERT(m_cache != nullptr); - m_cache->Read(offset, buffer, size); - } - - void Write(s64 offset, const void *buffer, size_t size) { - AMS_ASSERT(m_cache != nullptr); - m_cache->Write(offset, buffer, size); - } - - Result Flush() { - AMS_ASSERT(m_cache != nullptr); - return m_cache->Flush(); - } - - void Invalidate() { - AMS_ASSERT(m_cache != nullptr); - return m_cache->Invalidate(); - } - - bool Hits(s64 offset, s64 size) const { - AMS_ASSERT(m_cache != nullptr); - return m_cache->Hits(offset, size); - } - private: - void Release() { - if (m_cache != nullptr) { - AMS_ASSERT(m_buffered_storage->m_caches.get() <= m_cache); - AMS_ASSERT(m_cache <= m_buffered_storage->m_caches.get() + m_buffered_storage->m_cache_count); - - m_cache->Link(); - m_cache = nullptr; - } - } - }; - - class BufferedStorage::UniqueCache { - NON_COPYABLE(UniqueCache); - NON_MOVEABLE(UniqueCache); - private: - Cache *m_cache; - BufferedStorage *m_buffered_storage; - public: - explicit UniqueCache(BufferedStorage *bs) : m_cache(nullptr), m_buffered_storage(bs) { - AMS_ASSERT(m_buffered_storage != nullptr); - } - - ~UniqueCache() { - if (m_cache != nullptr) { - std::scoped_lock lk(m_buffered_storage->m_mutex); - m_cache->UnprepareFetch(); - } - } - - const std::pair Upgrade(const SharedCache &shared_cache) { - AMS_ASSERT(m_buffered_storage == shared_cache.m_buffered_storage); - AMS_ASSERT(shared_cache.m_cache != nullptr); - - std::scoped_lock lk(m_buffered_storage->m_mutex); - const auto result = shared_cache.m_cache->PrepareFetch(); - if (R_SUCCEEDED(result.first) && result.second) { - m_cache = shared_cache.m_cache; - } - return result; - } - - Result Fetch(s64 offset) { - AMS_ASSERT(m_cache != nullptr); - return m_cache->Fetch(offset); - } - - Result FetchFromBuffer(s64 offset, const void *buffer, size_t buffer_size) { - AMS_ASSERT(m_cache != nullptr); - R_TRY(m_cache->FetchFromBuffer(offset, buffer, buffer_size)); - return ResultSuccess(); - } - }; - - BufferedStorage::BufferedStorage() : m_base_storage(), m_buffer_manager(), m_block_size(), m_base_storage_size(), m_caches(), m_cache_count(), m_next_acquire_cache(), m_next_fetch_cache(), m_mutex(), m_bulk_read_enabled() { - /* ... */ - } - - BufferedStorage::~BufferedStorage() { - this->Finalize(); - } - - Result BufferedStorage::Initialize(fs::SubStorage base_storage, fs::IBufferManager *buffer_manager, size_t block_size, s32 buffer_count) { - AMS_ASSERT(buffer_manager != nullptr); - AMS_ASSERT(block_size > 0); - AMS_ASSERT(util::IsPowerOfTwo(block_size)); - AMS_ASSERT(buffer_count > 0); - - /* Get the base storage size. */ - R_TRY(base_storage.GetSize(std::addressof(m_base_storage_size))); - - /* Set members. */ - m_base_storage = base_storage; - m_buffer_manager = buffer_manager; - m_block_size = block_size; - m_cache_count = buffer_count; - - /* Allocate the caches. */ - m_caches.reset(new Cache[buffer_count]); - R_UNLESS(m_caches != nullptr, fs::ResultAllocationFailureInBufferedStorageA()); - - /* Initialize the caches. */ - for (auto i = 0; i < buffer_count; i++) { - m_caches[i].Initialize(this); - } - - m_next_acquire_cache = std::addressof(m_caches[0]); - return ResultSuccess(); - } - - void BufferedStorage::Finalize() { - m_base_storage = fs::SubStorage(); - m_base_storage_size = 0; - m_caches.reset(); - m_cache_count = 0; - m_next_fetch_cache = nullptr; - } - - Result BufferedStorage::Read(s64 offset, void *buffer, size_t size) { - AMS_ASSERT(this->IsInitialized()); - - /* Succeed if zero size. */ - R_SUCCEED_IF(size == 0); - - /* Validate arguments. */ - R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); - - /* Do the read. */ - R_TRY(this->ReadCore(offset, buffer, size)); - return ResultSuccess(); - } - - Result BufferedStorage::Write(s64 offset, const void *buffer, size_t size) { - AMS_ASSERT(this->IsInitialized()); - - /* Succeed if zero size. */ - R_SUCCEED_IF(size == 0); - - /* Validate arguments. */ - R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); - - /* Do the write. */ - R_TRY(this->WriteCore(offset, buffer, size)); - return ResultSuccess(); - } - - Result BufferedStorage::GetSize(s64 *out) { - AMS_ASSERT(out != nullptr); - AMS_ASSERT(this->IsInitialized()); - - *out = m_base_storage_size; - return ResultSuccess(); - } - - Result BufferedStorage::SetSize(s64 size) { - AMS_ASSERT(this->IsInitialized()); - const s64 prev_size = m_base_storage_size; - if (prev_size < size) { - /* Prepare to expand. */ - if (!util::IsAligned(prev_size, m_block_size)) { - SharedCache cache(this); - const auto invalidate_offset = prev_size; - const auto invalidate_size = size - prev_size; - if (cache.AcquireNextOverlappedCache(invalidate_offset, invalidate_size)) { - R_TRY(cache.Flush()); - cache.Invalidate(); - } - AMS_ASSERT(!cache.AcquireNextOverlappedCache(invalidate_offset, invalidate_size)); - } - } else if (size < prev_size) { - /* Prepare to do a shrink. */ - SharedCache cache(this); - const auto invalidate_offset = prev_size; - const auto invalidate_size = size - prev_size; - const auto is_fragment = util::IsAligned(size, m_block_size); - while (cache.AcquireNextOverlappedCache(invalidate_offset, invalidate_size)) { - if (is_fragment && cache.Hits(invalidate_offset, 1)) { - R_TRY(cache.Flush()); - } - cache.Invalidate(); - } - } - - /* Set the size. */ - R_TRY(m_base_storage.SetSize(size)); - - /* Get our new size. */ - s64 new_size = 0; - R_TRY(m_base_storage.GetSize(std::addressof(new_size))); - - m_base_storage_size = new_size; - return ResultSuccess(); - } - - Result BufferedStorage::Flush() { - AMS_ASSERT(this->IsInitialized()); - - /* Flush caches. */ - SharedCache cache(this); - while (cache.AcquireNextDirtyCache()) { - R_TRY(cache.Flush()); - } - - /* Flush the base storage. */ - R_TRY(m_base_storage.Flush()); - return ResultSuccess(); - } - - Result BufferedStorage::OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) { - AMS_ASSERT(this->IsInitialized()); - - /* Invalidate caches, if we should. */ - if (op_id == fs::OperationId::Invalidate) { - SharedCache cache(this); - while (cache.AcquireNextOverlappedCache(offset, size)) { - cache.Invalidate(); - } - } - - return m_base_storage.OperateRange(dst, dst_size, op_id, offset, size, src, src_size); - } - - void BufferedStorage::InvalidateCaches() { - AMS_ASSERT(this->IsInitialized()); - - SharedCache cache(this); - while (cache.AcquireNextValidCache()) { - cache.Invalidate(); - } - } - - Result BufferedStorage::PrepareAllocation() { - const auto flush_threshold = m_buffer_manager->GetTotalSize() / 8; - if (m_buffer_manager->GetTotalAllocatableSize() < flush_threshold) { - R_TRY(this->Flush()); - } - return ResultSuccess(); - } - - Result BufferedStorage::ControlDirtiness() { - const auto flush_threshold = m_buffer_manager->GetTotalSize() / 4; - if (m_buffer_manager->GetTotalAllocatableSize() < flush_threshold) { - s32 dirty_count = 0; - SharedCache cache(this); - while (cache.AcquireNextDirtyCache()) { - if ((++dirty_count) > 1) { - R_TRY(cache.Flush()); - cache.Invalidate(); - } - } - } - return ResultSuccess(); - } - - Result BufferedStorage::ReadCore(s64 offset, void *buffer, size_t size) { - AMS_ASSERT(m_caches != nullptr); - AMS_ASSERT(buffer != nullptr); - - /* Validate the offset. */ - const auto base_storage_size = m_base_storage_size; - R_UNLESS(offset >= 0, fs::ResultInvalidOffset()); - R_UNLESS(offset <= base_storage_size, fs::ResultInvalidOffset()); - - /* Setup tracking variables. */ - size_t remaining_size = static_cast(std::min(size, base_storage_size - offset)); - s64 cur_offset = offset; - s64 buf_offset = 0; - - /* Determine what caches are needed, if we have bulk read set. */ - if (m_bulk_read_enabled) { - /* Check head cache. */ - const auto head_cache_needed = this->ReadHeadCache(std::addressof(cur_offset), buffer, std::addressof(remaining_size), std::addressof(buf_offset)); - R_SUCCEED_IF(remaining_size == 0); - - /* Check tail cache. */ - const auto tail_cache_needed = this->ReadTailCache(cur_offset, buffer, std::addressof(remaining_size), buf_offset); - R_SUCCEED_IF(remaining_size == 0); - - /* Perform bulk reads. */ - constexpr size_t BulkReadSizeMax = 2_MB; - if (remaining_size <= BulkReadSizeMax) { - do { - /* Try to do a bulk read. */ - R_TRY_CATCH(this->BulkRead(cur_offset, static_cast(buffer) + buf_offset, remaining_size, head_cache_needed, tail_cache_needed)) { - R_CATCH(fs::ResultAllocationFailurePooledBufferNotEnoughSize) { - /* If the read fails due to insufficient pooled buffer size, */ - /* then we want to fall back to the normal read path. */ - break; - } - } R_END_TRY_CATCH; - - return ResultSuccess(); - } while(0); - } - } - - /* Repeatedly read until we're done. */ - while (remaining_size > 0) { - /* Determine how much to read this iteration. */ - auto *cur_dst = static_cast(buffer) + buf_offset; - size_t cur_size = 0; - - if (!util::IsAligned(cur_offset, m_block_size)) { - const size_t aligned_size = m_block_size - (cur_offset & (m_block_size - 1)); - cur_size = std::min(aligned_size, remaining_size); - } else if (remaining_size < m_block_size) { - cur_size = remaining_size; - } else { - cur_size = util::AlignDown(remaining_size, m_block_size); - } - - if (cur_size <= m_block_size) { - SharedCache cache(this); - if (!cache.AcquireNextOverlappedCache(cur_offset, cur_size)) { - R_TRY(this->PrepareAllocation()); - while (true) { - R_UNLESS(cache.AcquireFetchableCache(), fs::ResultOutOfResource()); - - UniqueCache fetch_cache(this); - const auto upgrade_result = fetch_cache.Upgrade(cache); - R_TRY(upgrade_result.first); - if (upgrade_result.second) { - R_TRY(fetch_cache.Fetch(cur_offset)); - break; - } - } - R_TRY(this->ControlDirtiness()); - } - cache.Read(cur_offset, cur_dst, cur_size); - } else { - { - SharedCache cache(this); - while (cache.AcquireNextOverlappedCache(cur_offset, cur_size)) { - R_TRY(cache.Flush()); - cache.Invalidate(); - } - } - R_TRY(m_base_storage.Read(cur_offset, cur_dst, cur_size)); - } - - remaining_size -= cur_size; - cur_offset += cur_size; - buf_offset += cur_size; - } - - return ResultSuccess(); - } - - bool BufferedStorage::ReadHeadCache(s64 *offset, void *buffer, size_t *size, s64 *buffer_offset) { - AMS_ASSERT(offset != nullptr); - AMS_ASSERT(buffer != nullptr); - AMS_ASSERT(size != nullptr); - AMS_ASSERT(buffer_offset != nullptr); - - bool is_cache_needed = !util::IsAligned(*offset, m_block_size); - - while (*size > 0) { - size_t cur_size = 0; - - if (!util::IsAligned(*offset, m_block_size)) { - const s64 aligned_size = util::AlignUp(*offset, m_block_size) - *offset; - cur_size = std::min(aligned_size, static_cast(*size)); - } else if (*size < m_block_size) { - cur_size = *size; - } else { - cur_size = m_block_size; - } - - SharedCache cache(this); - if (!cache.AcquireNextOverlappedCache(*offset, cur_size)) { - break; - } - - cache.Read(*offset, static_cast(buffer) + *buffer_offset, cur_size); - *offset += cur_size; - *buffer_offset += cur_size; - *size -= cur_size; - is_cache_needed = false; - } - - return is_cache_needed; - } - - bool BufferedStorage::ReadTailCache(s64 offset, void *buffer, size_t *size, s64 buffer_offset) { - AMS_ASSERT(buffer != nullptr); - AMS_ASSERT(size != nullptr); - - bool is_cache_needed = !util::IsAligned(offset + *size, m_block_size); - - while (*size > 0) { - const s64 cur_offset_end = offset + *size; - size_t cur_size = 0; - - if (!util::IsAligned(cur_offset_end, m_block_size)) { - const s64 aligned_size = cur_offset_end - util::AlignDown(cur_offset_end, m_block_size); - cur_size = std::min(aligned_size, static_cast(*size)); - } else if (*size < m_block_size) { - cur_size = *size; - } else { - cur_size = m_block_size; - } - - const s64 cur_offset = cur_offset_end - static_cast(cur_size); - AMS_ASSERT(cur_offset >= 0); - - SharedCache cache(this); - if (!cache.AcquireNextOverlappedCache(cur_offset, cur_size)) { - break; - } - - cache.Read(cur_offset, static_cast(buffer) + buffer_offset + cur_offset - offset, cur_size); - *size -= cur_size; - is_cache_needed = false; - } - - return is_cache_needed; - } - - Result BufferedStorage::BulkRead(s64 offset, void *buffer, size_t size, bool head_cache_needed, bool tail_cache_needed) { - /* Determine aligned extents. */ - const s64 aligned_offset = util::AlignDown(offset, m_block_size); - const s64 aligned_offset_end = std::min(util::AlignUp(offset + static_cast(size), m_block_size), m_base_storage_size); - const s64 aligned_size = aligned_offset_end - aligned_offset; - - /* Allocate a work buffer. */ - char *work_buffer = nullptr; - PooledBuffer pooled_buffer; - if (offset == aligned_offset && size == static_cast(aligned_size)) { - work_buffer = static_cast(buffer); - } else { - pooled_buffer.AllocateParticularlyLarge(static_cast(aligned_size), 1); - R_UNLESS(static_cast(pooled_buffer.GetSize()) >= aligned_size, fs::ResultAllocationFailurePooledBufferNotEnoughSize()); - work_buffer = pooled_buffer.GetBuffer(); - } - - /* Ensure cache is coherent. */ - { - SharedCache cache(this); - while (cache.AcquireNextOverlappedCache(aligned_offset, aligned_size)) { - R_TRY(cache.Flush()); - cache.Invalidate(); - } - } - - /* Read from the base storage. */ - R_TRY(m_base_storage.Read(aligned_offset, work_buffer, static_cast(aligned_size))); - if (work_buffer != static_cast(buffer)) { - std::memcpy(buffer, work_buffer + offset - aligned_offset, size); - } - - bool cached = false; - - /* Handle head cache if needed. */ - if (head_cache_needed) { - R_TRY(this->PrepareAllocation()); - - SharedCache cache(this); - while (true) { - R_UNLESS(cache.AcquireFetchableCache(), fs::ResultOutOfResource()); - - UniqueCache fetch_cache(this); - const auto upgrade_result = fetch_cache.Upgrade(cache); - R_TRY(upgrade_result.first); - if (upgrade_result.second) { - R_TRY(fetch_cache.FetchFromBuffer(aligned_offset, work_buffer, static_cast(aligned_size))); - break; - } - } - - cached = true; - } - - /* Handle tail cache if needed. */ - if (tail_cache_needed && (!head_cache_needed || aligned_size > static_cast(m_block_size))) { - if (!cached) { - R_TRY(this->PrepareAllocation()); - } - - SharedCache cache(this); - while (true) { - R_UNLESS(cache.AcquireFetchableCache(), fs::ResultOutOfResource()); - - UniqueCache fetch_cache(this); - const auto upgrade_result = fetch_cache.Upgrade(cache); - R_TRY(upgrade_result.first); - if (upgrade_result.second) { - const s64 tail_cache_offset = util::AlignDown(offset + static_cast(size), m_block_size); - const size_t tail_cache_size = static_cast(aligned_size - tail_cache_offset + aligned_offset); - R_TRY(fetch_cache.FetchFromBuffer(tail_cache_offset, work_buffer + tail_cache_offset - aligned_offset, tail_cache_size)); - break; - } - } - } - - if (cached) { - R_TRY(this->ControlDirtiness()); - } - - return ResultSuccess(); - } - - Result BufferedStorage::WriteCore(s64 offset, const void *buffer, size_t size) { - AMS_ASSERT(m_caches != nullptr); - AMS_ASSERT(buffer != nullptr); - - /* Validate the offset. */ - const auto base_storage_size = m_base_storage_size; - R_UNLESS(offset >= 0, fs::ResultInvalidOffset()); - R_UNLESS(offset <= base_storage_size, fs::ResultInvalidOffset()); - - /* Setup tracking variables. */ - size_t remaining_size = static_cast(std::min(size, base_storage_size - offset)); - s64 cur_offset = offset; - s64 buf_offset = 0; - - /* Repeatedly read until we're done. */ - while (remaining_size > 0) { - /* Determine how much to read this iteration. */ - const auto *cur_src = static_cast(buffer) + buf_offset; - size_t cur_size = 0; - - if (!util::IsAligned(cur_offset, m_block_size)) { - const size_t aligned_size = m_block_size - (cur_offset & (m_block_size - 1)); - cur_size = std::min(aligned_size, remaining_size); - } else if (remaining_size < m_block_size) { - cur_size = remaining_size; - } else { - cur_size = util::AlignDown(remaining_size, m_block_size); - } - - if (cur_size <= m_block_size) { - SharedCache cache(this); - if (!cache.AcquireNextOverlappedCache(cur_offset, cur_size)) { - R_TRY(this->PrepareAllocation()); - while (true) { - R_UNLESS(cache.AcquireFetchableCache(), fs::ResultOutOfResource()); - - UniqueCache fetch_cache(this); - const auto upgrade_result = fetch_cache.Upgrade(cache); - R_TRY(upgrade_result.first); - if (upgrade_result.second) { - R_TRY(fetch_cache.Fetch(cur_offset)); - break; - } - } - } - cache.Write(cur_offset, cur_src, cur_size); - - buffers::EnableBlockingBufferManagerAllocation(); - - R_TRY(this->ControlDirtiness()); - } else { - { - SharedCache cache(this); - while (cache.AcquireNextOverlappedCache(cur_offset, cur_size)) { - R_TRY(cache.Flush()); - cache.Invalidate(); - } - } - - R_TRY(m_base_storage.Write(cur_offset, cur_src, cur_size)); - - buffers::EnableBlockingBufferManagerAllocation(); - } - - remaining_size -= cur_size; - cur_offset += cur_size; - buf_offset += cur_size; - } - - return ResultSuccess(); - } - -} diff --git a/libraries/libstratosphere/source/fssystem/save/fssystem_hierarchical_integrity_verification_storage.cpp b/libraries/libstratosphere/source/fssystem/save/fssystem_hierarchical_integrity_verification_storage.cpp deleted file mode 100644 index 80f46c3..0000000 --- a/libraries/libstratosphere/source/fssystem/save/fssystem_hierarchical_integrity_verification_storage.cpp +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Copyright (c) Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#include - -namespace ams::fssystem::save { - - namespace { - - constexpr inline u32 IntegrityVerificationStorageMagic = util::FourCC<'I','V','F','C'>::Code; - constexpr inline u32 IntegrityVerificationStorageVersion = 0x00020000; - constexpr inline u32 IntegrityVerificationStorageVersionMask = 0xFFFF0000; - - constexpr inline auto MaxSaveDataFsDataCacheEntryCount = 32; - constexpr inline auto MaxSaveDataFsHashCacheEntryCount = 4; - constexpr inline auto MaxRomFsDataCacheEntryCount = 24; - constexpr inline auto MaxRomFsHashCacheEntryCount = 8; - - constexpr inline auto AccessCountMax = 5; - constexpr inline auto AccessTimeout = TimeSpan::FromMilliSeconds(10); - - os::Semaphore g_read_semaphore(AccessCountMax, AccessCountMax); - os::Semaphore g_write_semaphore(AccessCountMax, AccessCountMax); - - constexpr inline const char MasterKey[] = "HierarchicalIntegrityVerificationStorage::Master"; - constexpr inline const char L1Key[] = "HierarchicalIntegrityVerificationStorage::L1"; - constexpr inline const char L2Key[] = "HierarchicalIntegrityVerificationStorage::L2"; - constexpr inline const char L3Key[] = "HierarchicalIntegrityVerificationStorage::L3"; - constexpr inline const char L4Key[] = "HierarchicalIntegrityVerificationStorage::L4"; - constexpr inline const char L5Key[] = "HierarchicalIntegrityVerificationStorage::L5"; - - constexpr inline const struct { - const char *key; - size_t size; - } KeyArray[] = { - { MasterKey, sizeof(MasterKey) }, - { L1Key, sizeof(L1Key) }, - { L2Key, sizeof(L2Key) }, - { L3Key, sizeof(L3Key) }, - { L4Key, sizeof(L4Key) }, - { L5Key, sizeof(L5Key) }, - }; - - } - - /* Instantiate the global random generation function. */ - constinit HierarchicalIntegrityVerificationStorage::GenerateRandomFunction HierarchicalIntegrityVerificationStorage::s_generate_random = nullptr; - - Result HierarchicalIntegrityVerificationStorageControlArea::QuerySize(HierarchicalIntegrityVerificationSizeSet *out, const InputParam &input_param, s32 layer_count, s64 data_size) { - /* Validate preconditions. */ - AMS_ASSERT(out != nullptr); - AMS_ASSERT((static_cast(IntegrityMinLayerCount) <= layer_count) && (layer_count <= static_cast(IntegrityMaxLayerCount))); - for (s32 level = 0; level < (layer_count - 1); ++level) { - AMS_ASSERT(input_param.level_block_size[level] > 0); - AMS_ASSERT(IsPowerOfTwo(static_cast(input_param.level_block_size[level]))); - } - - /* Set the control size. */ - out->control_size = sizeof(HierarchicalIntegrityVerificationMetaInformation); - - /* Determine the level sizes. */ - s64 level_size[IntegrityMaxLayerCount]; - s32 level = layer_count - 1; - - level_size[level] = util::AlignUp(data_size, input_param.level_block_size[level - 1]); - --level; - - for (/* ... */; level > 0; --level) { - level_size[level] = util::AlignUp(level_size[level + 1] / input_param.level_block_size[level] * HashSize, input_param.level_block_size[level - 1]); - } - - /* Determine the master size. */ - level_size[0] = level_size[1] / input_param.level_block_size[0] * HashSize; - - /* Set the master size. */ - out->master_hash_size = level_size[0]; - - /* Set the level sizes. */ - for (level = 1; level < layer_count - 1; ++level) { - out->layered_hash_sizes[level - 1] = level_size[level]; - } - - return ResultSuccess(); - } - - Result HierarchicalIntegrityVerificationStorageControlArea::Expand(fs::SubStorage meta_storage, const HierarchicalIntegrityVerificationMetaInformation &meta) { - /* Check the meta size. */ - { - s64 meta_size = 0; - R_TRY(meta_storage.GetSize(std::addressof(meta_size))); - R_UNLESS(meta_size >= static_cast(sizeof(meta)), fs::ResultInvalidSize()); - } - - /* Validate both the previous and new metas. */ - { - /* Read the previous meta. */ - HierarchicalIntegrityVerificationMetaInformation prev_meta = {}; - R_TRY(meta_storage.Read(0, std::addressof(prev_meta), sizeof(prev_meta))); - - /* Validate both magics. */ - R_UNLESS(prev_meta.magic == IntegrityVerificationStorageMagic, fs::ResultIncorrectIntegrityVerificationMagic()); - R_UNLESS(prev_meta.magic == meta.magic, fs::ResultIncorrectIntegrityVerificationMagic()); - - /* Validate both versions. */ - R_UNLESS(prev_meta.version == IntegrityVerificationStorageVersion, fs::ResultUnsupportedVersion()); - R_UNLESS(prev_meta.version == meta.version, fs::ResultUnsupportedVersion()); - } - - /* Write the new meta. */ - R_TRY(meta_storage.Write(0, std::addressof(meta), sizeof(meta))); - R_TRY(meta_storage.Flush()); - - return ResultSuccess(); - } - - Result HierarchicalIntegrityVerificationStorageControlArea::Initialize(fs::SubStorage meta_storage) { - /* Check the meta size. */ - { - s64 meta_size = 0; - R_TRY(meta_storage.GetSize(std::addressof(meta_size))); - R_UNLESS(meta_size >= static_cast(sizeof(m_meta)), fs::ResultInvalidSize()); - } - - /* Set the storage and read the meta. */ - m_storage = meta_storage; - R_TRY(m_storage.Read(0, std::addressof(m_meta), sizeof(m_meta))); - - /* Validate the meta magic. */ - R_UNLESS(m_meta.magic == IntegrityVerificationStorageMagic, fs::ResultIncorrectIntegrityVerificationMagic()); - - /* Validate the meta version. */ - R_UNLESS((m_meta.version & IntegrityVerificationStorageVersionMask) == (IntegrityVerificationStorageVersion & IntegrityVerificationStorageVersionMask), fs::ResultUnsupportedVersion()); - - return ResultSuccess(); - } - - void HierarchicalIntegrityVerificationStorageControlArea::Finalize() { - m_storage = fs::SubStorage(); - } - - Result HierarchicalIntegrityVerificationStorage::Initialize(const HierarchicalIntegrityVerificationInformation &info, HierarchicalStorageInformation storage, FileSystemBufferManagerSet *bufs, IHash256GeneratorFactory *hgf, os::SdkRecursiveMutex *mtx, fs::StorageType storage_type) { - /* Validate preconditions. */ - AMS_ASSERT(bufs != nullptr); - AMS_ASSERT(IntegrityMinLayerCount <= info.max_layers && info.max_layers <= IntegrityMaxLayerCount); - - /* Set member variables. */ - m_max_layers = info.max_layers; - m_buffers = bufs; - m_mutex = mtx; - - /* Determine our cache counts. */ - const auto max_data_cache_entry_count = (storage_type == fs::StorageType_SaveData) ? MaxSaveDataFsDataCacheEntryCount : MaxRomFsDataCacheEntryCount; - const auto max_hash_cache_entry_count = (storage_type == fs::StorageType_SaveData) ? MaxSaveDataFsHashCacheEntryCount : MaxRomFsHashCacheEntryCount; - - /* Initialize the top level verification storage. */ - { - fs::HashSalt mac; - crypto::GenerateHmacSha256(mac.value, sizeof(mac), info.seed.value, sizeof(info.seed), KeyArray[0].key, KeyArray[0].size); - m_verify_storages[0].Initialize(storage[HierarchicalStorageInformation::MasterStorage], storage[HierarchicalStorageInformation::Layer1Storage], static_cast(1) << info.info[0].block_order, HashSize, m_buffers->buffers[m_max_layers - 2], hgf, mac, false, storage_type); - } - - /* Ensure we don't leak state if further initialization goes wrong. */ - auto top_verif_guard = SCOPE_GUARD { - m_verify_storages[0].Finalize(); - - m_data_size = -1; - m_buffers = nullptr; - m_mutex = nullptr; - }; - - /* Initialize the top level buffer storage. */ - R_TRY(m_buffer_storages[0].Initialize(m_buffers->buffers[0], m_mutex, std::addressof(m_verify_storages[0]), info.info[0].size, static_cast(1) << info.info[0].block_order, max_hash_cache_entry_count, false, 0x10, false, storage_type)); - auto top_buffer_guard = SCOPE_GUARD { m_buffer_storages[0].Finalize(); }; - - /* Prepare to initialize the level storages. */ - s32 level = 0; - - /* Ensure we don't leak state if further initialization goes wrong. */ - auto level_guard = SCOPE_GUARD { - m_verify_storages[level + 1].Finalize(); - for (/* ... */; level > 0; --level) { - m_buffer_storages[level].Finalize(); - m_verify_storages[level].Finalize(); - } - }; - - /* Initialize the level storages. */ - for (/* ... */; level < m_max_layers - 3; ++level) { - /* Initialize the verification storage. */ - { - fs::SubStorage buffer_storage(std::addressof(m_buffer_storages[level]), 0, info.info[level].size); - fs::HashSalt mac; - crypto::GenerateHmacSha256(mac.value, sizeof(mac), info.seed.value, sizeof(info.seed), KeyArray[level + 1].key, KeyArray[level + 1].size); - m_verify_storages[level + 1].Initialize(buffer_storage, storage[level + 2], static_cast(1) << info.info[level + 1].block_order, static_cast(1) << info.info[level].block_order, m_buffers->buffers[m_max_layers - 2], hgf, mac, false, storage_type); - } - - /* Initialize the buffer storage. */ - R_TRY(m_buffer_storages[level + 1].Initialize(m_buffers->buffers[level + 1], m_mutex, std::addressof(m_verify_storages[level + 1]), info.info[level + 1].size, static_cast(1) << info.info[level + 1].block_order, max_hash_cache_entry_count, false, 0x11 + static_cast(level), false, storage_type)); - } - - /* Initialize the final level storage. */ - { - /* Initialize the verification storage. */ - { - fs::SubStorage buffer_storage(std::addressof(m_buffer_storages[level]), 0, info.info[level].size); - fs::HashSalt mac; - crypto::GenerateHmacSha256(mac.value, sizeof(mac), info.seed.value, sizeof(info.seed), KeyArray[level + 1].key, KeyArray[level + 1].size); - m_verify_storages[level + 1].Initialize(buffer_storage, storage[level + 2], static_cast(1) << info.info[level + 1].block_order, static_cast(1) << info.info[level].block_order, m_buffers->buffers[m_max_layers - 2], hgf, mac, true, storage_type); - } - - /* Initialize the buffer storage. */ - R_TRY(m_buffer_storages[level + 1].Initialize(m_buffers->buffers[level + 1], m_mutex, std::addressof(m_verify_storages[level + 1]), info.info[level + 1].size, static_cast(1) << info.info[level + 1].block_order, max_data_cache_entry_count, true, 0x11 + static_cast(level), true, storage_type)); - } - - /* Set the data size. */ - m_data_size = info.info[level + 1].size; - - /* We succeeded. */ - level_guard.Cancel(); - top_buffer_guard.Cancel(); - top_verif_guard.Cancel(); - return ResultSuccess(); - } - - void HierarchicalIntegrityVerificationStorage::Finalize() { - if (m_data_size >= 0) { - m_data_size = 0; - - m_buffers = nullptr; - m_mutex = nullptr; - - for (s32 level = m_max_layers - 2; level >= 0; --level) { - m_buffer_storages[level].Finalize(); - m_verify_storages[level].Finalize(); - } - - m_data_size = -1; - } - } - - Result HierarchicalIntegrityVerificationStorage::Read(s64 offset, void *buffer, size_t size) { - /* Validate preconditions. */ - AMS_ASSERT(m_data_size >= 0); - - /* Succeed if zero-size. */ - R_SUCCEED_IF(size == 0); - - /* Validate arguments. */ - R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); - - /* Acquire access to the read semaphore. */ - if (!g_read_semaphore.TimedAcquire(AccessTimeout)) { - for (auto level = m_max_layers - 2; level >= 0; --level) { - R_TRY(m_buffer_storages[level].Flush()); - } - g_read_semaphore.Acquire(); - } - - /* Ensure that we release the semaphore when done. */ - ON_SCOPE_EXIT { g_read_semaphore.Release(); }; - - /* Read the data. */ - R_TRY(m_buffer_storages[m_max_layers - 2].Read(offset, buffer, size)); - return ResultSuccess(); - } - - Result HierarchicalIntegrityVerificationStorage::Write(s64 offset, const void *buffer, size_t size) { - /* Validate preconditions. */ - AMS_ASSERT(m_data_size >= 0); - - /* Succeed if zero-size. */ - R_SUCCEED_IF(size == 0); - - /* Validate arguments. */ - R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); - - /* Acquire access to the write semaphore. */ - if (!g_write_semaphore.TimedAcquire(AccessTimeout)) { - for (auto level = m_max_layers - 2; level >= 0; --level) { - R_TRY(m_buffer_storages[level].Flush()); - } - g_write_semaphore.Acquire(); - } - - /* Ensure that we release the semaphore when done. */ - ON_SCOPE_EXIT { g_write_semaphore.Release(); }; - - /* Write the data. */ - R_TRY(m_buffer_storages[m_max_layers - 2].Write(offset, buffer, size)); - m_is_written_for_rollback = true; - return ResultSuccess(); - } - - Result HierarchicalIntegrityVerificationStorage::GetSize(s64 *out) { - AMS_ASSERT(out != nullptr); - AMS_ASSERT(m_data_size >= 0); - *out = m_data_size; - return ResultSuccess(); - } - - Result HierarchicalIntegrityVerificationStorage::Flush() { - return ResultSuccess(); - } - - Result HierarchicalIntegrityVerificationStorage::OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) { - switch (op_id) { - case fs::OperationId::FillZero: - case fs::OperationId::DestroySignature: - { - R_TRY(m_buffer_storages[m_max_layers - 2].OperateRange(dst, dst_size, op_id, offset, size, src, src_size)); - m_is_written_for_rollback = true; - return ResultSuccess(); - } - case fs::OperationId::Invalidate: - case fs::OperationId::QueryRange: - { - R_TRY(m_buffer_storages[m_max_layers - 2].OperateRange(dst, dst_size, op_id, offset, size, src, src_size)); - return ResultSuccess(); - } - default: - return fs::ResultUnsupportedOperationInHierarchicalIntegrityVerificationStorageB(); - } - } - - Result HierarchicalIntegrityVerificationStorage::Commit() { - for (s32 level = m_max_layers - 2; level >= 0; --level) { - R_TRY(m_buffer_storages[level].Commit()); - } - return ResultSuccess(); - } - - Result HierarchicalIntegrityVerificationStorage::OnRollback() { - for (s32 level = m_max_layers - 2; level >= 0; --level) { - R_TRY(m_buffer_storages[level].OnRollback()); - } - m_is_written_for_rollback = false; - return ResultSuccess(); - } - -} diff --git a/libraries/libstratosphere/source/fssystem/save/fssystem_integrity_verification_storage.cpp b/libraries/libstratosphere/source/fssystem/save/fssystem_integrity_verification_storage.cpp deleted file mode 100644 index 6910ecf..0000000 --- a/libraries/libstratosphere/source/fssystem/save/fssystem_integrity_verification_storage.cpp +++ /dev/null @@ -1,487 +0,0 @@ -/* - * Copyright (c) Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#include - -namespace ams::fssystem::save { - - Result IntegrityVerificationStorage::Initialize(fs::SubStorage hs, fs::SubStorage ds, s64 verif_block_size, s64 upper_layer_verif_block_size, fs::IBufferManager *bm, fssystem::IHash256GeneratorFactory *hgf, const fs::HashSalt &salt, bool is_real_data, fs::StorageType storage_type) { - /* Validate preconditions. */ - AMS_ASSERT(verif_block_size >= HashSize); - AMS_ASSERT(bm != nullptr); - AMS_ASSERT(hgf != nullptr); - - /* Set storages. */ - m_hash_storage = hs; - m_data_storage = ds; - - /* Set hash generator factory. */ - m_hash_generator_factory = hgf; - - /* Set verification block sizes. */ - m_verification_block_size = verif_block_size; - m_verification_block_order = ILog2(static_cast(verif_block_size)); - AMS_ASSERT(m_verification_block_size == (1l << m_verification_block_order)); - - /* Set buffer manager. */ - m_buffer_manager = bm; - - /* Set upper layer block sizes. */ - upper_layer_verif_block_size = std::max(upper_layer_verif_block_size, HashSize); - m_upper_layer_verification_block_size = upper_layer_verif_block_size; - m_upper_layer_verification_block_order = ILog2(static_cast(upper_layer_verif_block_size)); - AMS_ASSERT(m_upper_layer_verification_block_size == (1l << m_upper_layer_verification_block_order)); - - /* Validate sizes. */ - { - s64 hash_size = 0; - s64 data_size = 0; - AMS_ASSERT(R_SUCCEEDED(m_hash_storage.GetSize(std::addressof(hash_size)))); - AMS_ASSERT(R_SUCCEEDED(m_data_storage.GetSize(std::addressof(hash_size)))); - AMS_ASSERT(((hash_size / HashSize) * m_verification_block_size) >= data_size); - AMS_UNUSED(hash_size, data_size); - } - - /* Set salt. */ - std::memcpy(m_salt.value, salt.value, fs::HashSalt::Size); - - /* Set data and storage type. */ - m_is_real_data = is_real_data; - m_storage_type = storage_type; - return ResultSuccess(); - } - - void IntegrityVerificationStorage::Finalize() { - if (m_buffer_manager != nullptr) { - m_hash_storage = fs::SubStorage(); - m_data_storage = fs::SubStorage(); - m_buffer_manager = nullptr; - } - } - - Result IntegrityVerificationStorage::Read(s64 offset, void *buffer, size_t size) { - /* Although we support zero-size reads, we expect non-zero sizes. */ - AMS_ASSERT(size != 0); - - /* Validate other preconditions. */ - AMS_ASSERT(util::IsAligned(offset, static_cast(m_verification_block_size))); - AMS_ASSERT(util::IsAligned(size, static_cast(m_verification_block_size))); - - /* Succeed if zero size. */ - R_SUCCEED_IF(size == 0); - - /* Validate arguments. */ - R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); - - /* Validate the offset. */ - s64 data_size; - R_TRY(m_data_storage.GetSize(std::addressof(data_size))); - R_UNLESS(offset <= data_size, fs::ResultInvalidOffset()); - - /* Validate the access range. */ - R_UNLESS(IStorage::CheckAccessRange(offset, size, util::AlignUp(data_size, static_cast(m_verification_block_size))), fs::ResultOutOfRange()); - - /* Determine the read extents. */ - size_t read_size = size; - if (static_cast(offset + read_size) > data_size) { - /* Determine the padding sizes. */ - s64 padding_offset = data_size - offset; - size_t padding_size = static_cast(m_verification_block_size - (padding_offset & (m_verification_block_size - 1))); - AMS_ASSERT(static_cast(padding_size) < m_verification_block_size); - - /* Clear the padding. */ - std::memset(static_cast(buffer) + padding_offset, 0, padding_size); - - /* Set the new in-bounds size. */ - read_size = static_cast(data_size - offset); - } - - /* Perform the read. */ - { - auto clear_guard = SCOPE_GUARD { std::memset(buffer, 0, size); }; - R_TRY(m_data_storage.Read(offset, buffer, read_size)); - clear_guard.Cancel(); - } - - /* Verify the signatures. */ - Result verify_hash_result = ResultSuccess(); - - /* Create hash generator. */ - auto generator = m_hash_generator_factory->Create(); - - /* Prepare to validate the signatures. */ - const auto signature_count = size >> m_verification_block_order; - PooledBuffer signature_buffer(signature_count * sizeof(BlockHash), sizeof(BlockHash)); - const auto buffer_count = std::min(signature_count, signature_buffer.GetSize() / sizeof(BlockHash)); - - size_t verified_count = 0; - while (verified_count < signature_count) { - /* Read the current signatures. */ - const auto cur_count = std::min(buffer_count, signature_count - verified_count); - auto cur_result = this->ReadBlockSignature(signature_buffer.GetBuffer(), signature_buffer.GetSize(), offset + (verified_count << m_verification_block_order), cur_count << m_verification_block_order); - - /* Temporarily increase our priority. */ - ScopedThreadPriorityChanger cp(+1, ScopedThreadPriorityChanger::Mode::Relative); - - /* Loop over each signature we read. */ - for (size_t i = 0; i < cur_count && R_SUCCEEDED(cur_result); ++i) { - const auto verified_size = (verified_count + i) << m_verification_block_order; - u8 *cur_buf = static_cast(buffer) + verified_size; - cur_result = this->VerifyHash(cur_buf, reinterpret_cast(signature_buffer.GetBuffer()) + i, generator); - - /* If the data is corrupted, clear the corrupted parts. */ - if (fs::ResultIntegrityVerificationStorageCorrupted::Includes(cur_result)) { - std::memset(cur_buf, 0, m_verification_block_size); - - /* Set the result if we should. */ - if (!fs::ResultClearedRealDataVerificationFailed::Includes(cur_result) && m_storage_type != fs::StorageType_Authoring) { - verify_hash_result = cur_result; - } - - cur_result = ResultSuccess(); - } - } - - /* If we failed, clear and return. */ - if (R_FAILED(cur_result)) { - std::memset(buffer, 0, size); - return cur_result; - } - - /* Advance. */ - verified_count += cur_count; - } - - return verify_hash_result; - } - - Result IntegrityVerificationStorage::Write(s64 offset, const void *buffer, size_t size) { - /* Succeed if zero size. */ - R_SUCCEED_IF(size == 0); - - /* Validate arguments. */ - R_UNLESS(buffer != nullptr, fs::ResultNullptrArgument()); - R_UNLESS(IStorage::CheckOffsetAndSize(offset, size), fs::ResultInvalidOffset()); - - /* Validate the offset. */ - s64 data_size; - R_TRY(m_data_storage.GetSize(std::addressof(data_size))); - R_UNLESS(offset < data_size, fs::ResultInvalidOffset()); - - /* Validate the access range. */ - R_UNLESS(IStorage::CheckAccessRange(offset, size, util::AlignUp(data_size, static_cast(m_verification_block_size))), fs::ResultOutOfRange()); - - /* Validate preconditions. */ - AMS_ASSERT(util::IsAligned(offset, m_verification_block_size)); - AMS_ASSERT(util::IsAligned(size, m_verification_block_size)); - AMS_ASSERT(offset <= data_size); - AMS_ASSERT(static_cast(offset + size) < data_size + m_verification_block_size); - - /* Validate that if writing past the end, all extra data is zero padding. */ - if (static_cast(offset + size) > data_size) { - const u8 *padding_cur = static_cast(buffer) + data_size - offset; - const u8 *padding_end = padding_cur + (offset + size - data_size); - - while (padding_cur < padding_end) { - AMS_ASSERT((*padding_cur) == 0); - ++padding_cur; - } - } - - /* Determine the unpadded size to write. */ - auto write_size = size; - if (static_cast(offset + write_size) > data_size) { - write_size = static_cast(data_size - offset); - R_SUCCEED_IF(write_size == 0); - } - - /* Determine the size we're writing in blocks. */ - const auto aligned_write_size = util::AlignUp(write_size, m_verification_block_size); - - /* Write the updated block signatures. */ - Result update_result = ResultSuccess(); - size_t updated_count = 0; - { - const auto signature_count = aligned_write_size >> m_verification_block_order; - PooledBuffer signature_buffer(signature_count * sizeof(BlockHash), sizeof(BlockHash)); - const auto buffer_count = std::min(signature_count, signature_buffer.GetSize() / sizeof(BlockHash)); - - auto generator = m_hash_generator_factory->Create(); - - while (updated_count < signature_count) { - const auto cur_count = std::min(buffer_count, signature_count - updated_count); - - /* Calculate the hash with temporarily increased priority. */ - { - ScopedThreadPriorityChanger cp(+1, ScopedThreadPriorityChanger::Mode::Relative); - - for (size_t i = 0; i < cur_count; ++i) { - const auto updated_size = (updated_count + i) << m_verification_block_order; - this->CalcBlockHash(reinterpret_cast(signature_buffer.GetBuffer()) + i, reinterpret_cast(buffer) + updated_size, generator); - } - } - - /* Write the new block signatures. */ - if (R_FAILED((update_result = this->WriteBlockSignature(signature_buffer.GetBuffer(), signature_buffer.GetSize(), offset + (updated_count << m_verification_block_order), cur_count << m_verification_block_order)))) { - break; - } - - /* Advance. */ - updated_count += cur_count; - } - } - - /* Write the data. */ - R_TRY(m_data_storage.Write(offset, buffer, std::min(write_size, updated_count << m_verification_block_order))); - - return update_result; - } - - Result IntegrityVerificationStorage::GetSize(s64 *out) { - return m_data_storage.GetSize(out); - } - - Result IntegrityVerificationStorage::Flush() { - /* Flush both storages. */ - R_TRY(m_hash_storage.Flush()); - R_TRY(m_data_storage.Flush()); - return ResultSuccess(); - } - - Result IntegrityVerificationStorage::OperateRange(void *dst, size_t dst_size, fs::OperationId op_id, s64 offset, s64 size, const void *src, size_t src_size) { - /* Validate preconditions. */ - AMS_ASSERT(util::IsAligned(offset, static_cast(m_verification_block_size))); - AMS_ASSERT(util::IsAligned(size, static_cast(m_verification_block_size))); - - switch (op_id) { - case fs::OperationId::FillZero: - { - /* Clear should only be called for save data. */ - AMS_ASSERT(m_storage_type == fs::StorageType_SaveData); - - /* Validate the range. */ - s64 data_size = 0; - R_TRY(m_data_storage.GetSize(std::addressof(data_size))); - R_UNLESS(0 <= offset && offset <= data_size, fs::ResultInvalidOffset()); - - /* Determine the extents to clear. */ - const auto sign_offset = (offset >> m_verification_block_order) * HashSize; - const auto sign_size = (std::min(size, data_size - offset) >> m_verification_block_order) * HashSize; - - /* Allocate a work buffer. */ - const auto buf_size = static_cast(std::min(sign_size, static_cast(1) << (m_upper_layer_verification_block_order + 2))); - std::unique_ptr buf = fs::impl::MakeUnique(buf_size); - R_UNLESS(buf != nullptr, fs::ResultAllocationFailureInIntegrityVerificationStorageA()); - - /* Clear the work buffer. */ - std::memset(buf.get(), 0, buf_size); - - /* Clear in chunks. */ - auto remaining_size = sign_size; - - while (remaining_size > 0) { - const auto cur_size = static_cast(std::min(remaining_size, static_cast(buf_size))); - R_TRY(m_hash_storage.Write(sign_offset + sign_size - remaining_size, buf.get(), cur_size)); - remaining_size -= cur_size; - } - - return ResultSuccess(); - } - case fs::OperationId::DestroySignature: - { - /* Clear Signature should only be called for save data. */ - AMS_ASSERT(m_storage_type == fs::StorageType_SaveData); - - /* Validate the range. */ - s64 data_size = 0; - R_TRY(m_data_storage.GetSize(std::addressof(data_size))); - R_UNLESS(0 <= offset && offset <= data_size, fs::ResultInvalidOffset()); - - /* Determine the extents to clear the signature for. */ - const auto sign_offset = (offset >> m_verification_block_order) * HashSize; - const auto sign_size = (std::min(size, data_size - offset) >> m_verification_block_order) * HashSize; - - /* Allocate a work buffer. */ - std::unique_ptr buf = fs::impl::MakeUnique(sign_size); - R_UNLESS(buf != nullptr, fs::ResultAllocationFailureInIntegrityVerificationStorageB()); - - /* Read the existing signature. */ - R_TRY(m_hash_storage.Read(sign_offset, buf.get(), sign_size)); - - /* Clear the signature. */ - /* This sets all bytes to FF, with the verification bit cleared. */ - for (auto i = 0; i < sign_size; ++i) { - buf[i] ^= ((i + 1) % HashSize == 0 ? 0x7F : 0xFF); - } - - /* Write the cleared signature. */ - return m_hash_storage.Write(sign_offset, buf.get(), sign_size); - } - case fs::OperationId::Invalidate: - { - /* Only allow cache invalidation for RomFs. */ - R_UNLESS(m_storage_type != fs::StorageType_SaveData, fs::ResultUnsupportedOperationInIntegrityVerificationStorageB()); - - - /* Operate on our storages. */ - R_TRY(m_hash_storage.OperateRange(dst, dst_size, op_id, 0, std::numeric_limits::max(), src, src_size)); - R_TRY(m_data_storage.OperateRange(dst, dst_size, op_id, offset, size, src, src_size)); - - return ResultSuccess(); - } - case fs::OperationId::QueryRange: - { - /* Validate the range. */ - s64 data_size = 0; - R_TRY(m_data_storage.GetSize(std::addressof(data_size))); - R_UNLESS(0 <= offset && offset <= data_size, fs::ResultInvalidOffset()); - - /* Determine the real size to query. */ - const auto actual_size = std::min(size, data_size - offset); - - /* Query the data storage. */ - R_TRY(m_data_storage.OperateRange(dst, dst_size, op_id, offset, actual_size, src, src_size)); - - return ResultSuccess(); - } - default: - return fs::ResultUnsupportedOperationInIntegrityVerificationStorageC(); - } - } - - void IntegrityVerificationStorage::CalcBlockHash(BlockHash *out, const void *buffer, size_t block_size, std::unique_ptr &generator) const { - /* Initialize the generator. */ - generator->Initialize(); - - /* If calculating for save data, hash the salt. */ - if (m_storage_type == fs::StorageType_SaveData) { - generator->Update(m_salt.value, sizeof(m_salt)); - } - - /* Update with the buffer and get the hash. */ - generator->Update(buffer, block_size); - generator->GetHash(out, sizeof(*out)); - - /* Set the validation bit, if the hash is for save data. */ - if (m_storage_type == fs::StorageType_SaveData) { - SetValidationBit(out); - } - } - - Result IntegrityVerificationStorage::ReadBlockSignature(void *dst, size_t dst_size, s64 offset, size_t size) { - /* Validate preconditions. */ - AMS_ASSERT(dst != nullptr); - AMS_ASSERT(util::IsAligned(offset, static_cast(m_verification_block_size))); - AMS_ASSERT(util::IsAligned(size, static_cast(m_verification_block_size))); - - /* Determine where to read the signature. */ - const s64 sign_offset = (offset >> m_verification_block_order) * HashSize; - const auto sign_size = static_cast((size >> m_verification_block_order) * HashSize); - AMS_ASSERT(dst_size >= sign_size); - AMS_UNUSED(dst_size); - - /* Create a guard in the event of failure. */ - auto clear_guard = SCOPE_GUARD { std::memset(dst, 0, sign_size); }; - - /* Validate that we can read the signature. */ - s64 hash_size; - R_TRY(m_hash_storage.GetSize(std::addressof(hash_size))); - const bool range_valid = static_cast(sign_offset + sign_size) <= hash_size; - AMS_ASSERT(range_valid); - R_UNLESS(range_valid, fs::ResultOutOfRange()); - - /* Read the signature. */ - R_TRY(m_hash_storage.Read(sign_offset, dst, sign_size)); - - /* We succeeded. */ - clear_guard.Cancel(); - return ResultSuccess(); - } - - Result IntegrityVerificationStorage::WriteBlockSignature(const void *src, size_t src_size, s64 offset, size_t size) { - /* Validate preconditions. */ - AMS_ASSERT(src != nullptr); - AMS_ASSERT(util::IsAligned(offset, static_cast(m_verification_block_size))); - - /* Determine where to write the signature. */ - const s64 sign_offset = (offset >> m_verification_block_order) * HashSize; - const auto sign_size = static_cast((size >> m_verification_block_order) * HashSize); - AMS_ASSERT(src_size >= sign_size); - AMS_UNUSED(src_size); - - /* Write the signature. */ - R_TRY(m_hash_storage.Write(sign_offset, src, sign_size)); - - /* We succeeded. */ - return ResultSuccess(); - } - - Result IntegrityVerificationStorage::VerifyHash(const void *buf, BlockHash *hash, std::unique_ptr &generator) { - /* Validate preconditions. */ - AMS_ASSERT(buf != nullptr); - AMS_ASSERT(hash != nullptr); - - /* Get the comparison hash. */ - auto &cmp_hash = *hash; - - /* If save data, check if the data is uninitialized. */ - if (m_storage_type == fs::StorageType_SaveData) { - bool is_cleared = false; - R_TRY(this->IsCleared(std::addressof(is_cleared), cmp_hash)); - R_UNLESS(!is_cleared, fs::ResultClearedRealDataVerificationFailed()); - } - - /* Get the calculated hash. */ - BlockHash calc_hash; - this->CalcBlockHash(std::addressof(calc_hash), buf, generator); - - /* Check that the signatures are equal. */ - if (!crypto::IsSameBytes(std::addressof(cmp_hash), std::addressof(calc_hash), sizeof(BlockHash))) { - /* Clear the comparison hash. */ - std::memset(std::addressof(cmp_hash), 0, sizeof(cmp_hash)); - - /* Return the appropriate result. */ - if (m_is_real_data) { - return fs::ResultUnclearedRealDataVerificationFailed(); - } else { - return fs::ResultNonRealDataVerificationFailed(); - } - } - - return ResultSuccess(); - } - - Result IntegrityVerificationStorage::IsCleared(bool *is_cleared, const BlockHash &hash) { - /* Validate preconditions. */ - AMS_ASSERT(is_cleared != nullptr); - AMS_ASSERT(m_storage_type == fs::StorageType_SaveData); - - /* Default to uncleared. */ - *is_cleared = false; - - /* Succeed if the validation bit is set. */ - R_SUCCEED_IF(IsValidationBit(std::addressof(hash))); - - /* Otherwise, we expect the hash to be all zero. */ - for (size_t i = 0; i < sizeof(hash.hash); ++i) { - R_UNLESS(hash.hash[i] == 0, fs::ResultInvalidZeroHash()); - } - - /* Set cleared. */ - *is_cleared = true; - return ResultSuccess(); - } - -}