From 14e2df56101f7c7ab87939ea7a708ab4e6fb70c6 Mon Sep 17 00:00:00 2001 From: Lioncash Date: Tue, 25 Sep 2018 17:26:09 -0400 Subject: [PATCH 01/11] vfs_static: Remove template byte parameter from StaticVfsFile This converts it into a regular constructor parameter. There's no need to make this a template parameter on the class when it functions perfectly well as a constructor argument. This also reduces the amount of code bloat produced by the compiler, as it doesn't need to generate the same code for multiple different instantiations of the same class type, but with a different fill value. --- src/core/file_sys/romfs.cpp | 2 +- src/core/file_sys/vfs_concat.cpp | 42 +++++++++++++++++++++++++------- src/core/file_sys/vfs_concat.h | 33 +++---------------------- src/core/file_sys/vfs_static.h | 7 +++--- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/core/file_sys/romfs.cpp b/src/core/file_sys/romfs.cpp index 205284a4d1..7804ef56d1 100644 --- a/src/core/file_sys/romfs.cpp +++ b/src/core/file_sys/romfs.cpp @@ -134,7 +134,7 @@ VirtualFile CreateRomFS(VirtualDir dir) { return nullptr; RomFSBuildContext ctx{dir}; - return ConcatenateFiles<0>(ctx.Build(), dir->GetName()); + return ConcatenateFiles(0, ctx.Build(), dir->GetName()); } } // namespace FileSys diff --git a/src/core/file_sys/vfs_concat.cpp b/src/core/file_sys/vfs_concat.cpp index d9f9911da0..8a0df508e8 100644 --- a/src/core/file_sys/vfs_concat.cpp +++ b/src/core/file_sys/vfs_concat.cpp @@ -7,6 +7,7 @@ #include "common/assert.h" #include "core/file_sys/vfs_concat.h" +#include "core/file_sys/vfs_static.h" namespace FileSys { @@ -22,15 +23,6 @@ static bool VerifyConcatenationMapContinuity(const std::map& m return map.begin()->first == 0; } -VirtualFile ConcatenateFiles(std::vector files, std::string name) { - if (files.empty()) - return nullptr; - if (files.size() == 1) - return files[0]; - - return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), std::move(name))); -} - ConcatenatedVfsFile::ConcatenatedVfsFile(std::vector files_, std::string name) : name(std::move(name)) { std::size_t next_offset = 0; @@ -109,4 +101,36 @@ bool ConcatenatedVfsFile::Rename(std::string_view name) { return false; } +VirtualFile ConcatenateFiles(std::vector files, std::string name) { + if (files.empty()) + return nullptr; + if (files.size() == 1) + return files[0]; + + return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), std::move(name))); +} + +VirtualFile ConcatenateFiles(u8 filler_byte, std::map files, std::string name) { + if (files.empty()) + return nullptr; + if (files.size() == 1) + return files.begin()->second; + + const auto last_valid = --files.end(); + for (auto iter = files.begin(); iter != last_valid;) { + const auto old = iter++; + if (old->first + old->second->GetSize() != iter->first) { + files.emplace(old->first + old->second->GetSize(), + std::make_shared(filler_byte, iter->first - old->first - + old->second->GetSize())); + } + } + + // Ensure the map starts at offset 0 (start of file), otherwise pad to fill. + if (files.begin()->first != 0) + files.emplace(0, std::make_shared(filler_byte, files.begin()->first)); + + return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), std::move(name))); +} + } // namespace FileSys diff --git a/src/core/file_sys/vfs_concat.h b/src/core/file_sys/vfs_concat.h index 76211d38a3..17fa40adee 100644 --- a/src/core/file_sys/vfs_concat.h +++ b/src/core/file_sys/vfs_concat.h @@ -7,9 +7,7 @@ #include #include #include -#include #include "core/file_sys/vfs.h" -#include "core/file_sys/vfs_static.h" namespace FileSys { @@ -17,9 +15,8 @@ namespace FileSys { // read-only. class ConcatenatedVfsFile : public VfsFile { friend VirtualFile ConcatenateFiles(std::vector files, std::string name); - - template - friend VirtualFile ConcatenateFiles(std::map files, std::string name); + friend VirtualFile ConcatenateFiles(u8 filler_byte, std::map files, + std::string name); ConcatenatedVfsFile(std::vector files, std::string name); ConcatenatedVfsFile(std::map files, std::string name); @@ -47,29 +44,7 @@ private: VirtualFile ConcatenateFiles(std::vector files, std::string name); // Convenience function that turns a map of offsets to files into a concatenated file, filling gaps -// with template parameter. -template -VirtualFile ConcatenateFiles(std::map files, std::string name) { - if (files.empty()) - return nullptr; - if (files.size() == 1) - return files.begin()->second; - - const auto last_valid = --files.end(); - for (auto iter = files.begin(); iter != last_valid;) { - const auto old = iter++; - if (old->first + old->second->GetSize() != iter->first) { - files.emplace(old->first + old->second->GetSize(), - std::make_shared>(iter->first - old->first - - old->second->GetSize())); - } - } - - // Ensure the map starts at offset 0 (start of file), otherwise pad to fill. - if (files.begin()->first != 0) - files.emplace(0, std::make_shared>(files.begin()->first)); - - return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), std::move(name))); -} +// with a given filler byte. +VirtualFile ConcatenateFiles(u8 filler_byte, std::map files, std::string name); } // namespace FileSys diff --git a/src/core/file_sys/vfs_static.h b/src/core/file_sys/vfs_static.h index 4dd47ffcc1..8ad77d3008 100644 --- a/src/core/file_sys/vfs_static.h +++ b/src/core/file_sys/vfs_static.h @@ -12,11 +12,11 @@ namespace FileSys { -template class StaticVfsFile : public VfsFile { public: - explicit StaticVfsFile(size_t size = 0, std::string name = "", VirtualDir parent = nullptr) - : size(size), name(std::move(name)), parent(std::move(parent)) {} + explicit StaticVfsFile(u8 value, size_t size = 0, std::string name = "", + VirtualDir parent = nullptr) + : value{value}, size{size}, name{std::move(name)}, parent{std::move(parent)} {} std::string GetName() const override { return name; @@ -70,6 +70,7 @@ public: } private: + u8 value; size_t size; std::string name; VirtualDir parent; From 28bef31ea80478fe58bc4eeaf1b245005f15b36a Mon Sep 17 00:00:00 2001 From: Lioncash Date: Tue, 25 Sep 2018 17:38:16 -0400 Subject: [PATCH 02/11] vfs_concat/vfs_layered: Remove friend declarations from ConcatenatedVfsFile Given these are only added to the class to allow those functions to access the private constructor, it's a better approach to just make them static functions in the interface, to make the dependency explicit. --- src/core/core.cpp | 2 +- src/core/file_sys/patch_manager.cpp | 5 +- src/core/file_sys/registered_cache.cpp | 2 +- src/core/file_sys/romfs.cpp | 2 +- src/core/file_sys/vfs_concat.cpp | 67 ++++++++++++++------------ src/core/file_sys/vfs_concat.h | 19 +++----- src/core/file_sys/vfs_layered.cpp | 15 +++--- src/core/file_sys/vfs_layered.h | 8 ++- 8 files changed, 59 insertions(+), 61 deletions(-) diff --git a/src/core/core.cpp b/src/core/core.cpp index 50f0a42fb1..7666354dc7 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -64,7 +64,7 @@ FileSys::VirtualFile GetGameFileFromPath(const FileSys::VirtualFilesystem& vfs, if (concat.empty()) return nullptr; - return FileSys::ConcatenateFiles(concat, dir->GetName()); + return FileSys::ConcatenatedVfsFile::MakeConcatenatedFile(concat, dir->GetName()); } return vfs->OpenFile(path, FileSys::Mode::Read); diff --git a/src/core/file_sys/patch_manager.cpp b/src/core/file_sys/patch_manager.cpp index af3f9a78fe..561ad67a70 100644 --- a/src/core/file_sys/patch_manager.cpp +++ b/src/core/file_sys/patch_manager.cpp @@ -90,10 +90,9 @@ static void ApplyLayeredFS(VirtualFile& romfs, u64 title_id, ContentRecordType t layers.push_back(std::move(extracted)); - const auto layered = LayerDirectories(layers); - + auto layered = LayeredVfsDirectory::MakeLayeredDirectory(std::move(layers)); if (layered != nullptr) { - auto packed = CreateRomFS(layered); + auto packed = CreateRomFS(std::move(layered)); if (packed != nullptr) { LOG_INFO(Loader, " RomFS: LayeredFS patches applied successfully"); diff --git a/src/core/file_sys/registered_cache.cpp b/src/core/file_sys/registered_cache.cpp index 653ef2e7b9..e9b0406894 100644 --- a/src/core/file_sys/registered_cache.cpp +++ b/src/core/file_sys/registered_cache.cpp @@ -125,7 +125,7 @@ VirtualFile RegisteredCache::OpenFileOrDirectoryConcat(const VirtualDir& dir, if (concat.empty()) return nullptr; - file = FileSys::ConcatenateFiles(concat, concat.front()->GetName()); + file = ConcatenatedVfsFile::MakeConcatenatedFile(concat, concat.front()->GetName()); } return file; diff --git a/src/core/file_sys/romfs.cpp b/src/core/file_sys/romfs.cpp index 7804ef56d1..5910f7046b 100644 --- a/src/core/file_sys/romfs.cpp +++ b/src/core/file_sys/romfs.cpp @@ -134,7 +134,7 @@ VirtualFile CreateRomFS(VirtualDir dir) { return nullptr; RomFSBuildContext ctx{dir}; - return ConcatenateFiles(0, ctx.Build(), dir->GetName()); + return ConcatenatedVfsFile::MakeConcatenatedFile(0, ctx.Build(), dir->GetName()); } } // namespace FileSys diff --git a/src/core/file_sys/vfs_concat.cpp b/src/core/file_sys/vfs_concat.cpp index 8a0df508e8..16d801c0c9 100644 --- a/src/core/file_sys/vfs_concat.cpp +++ b/src/core/file_sys/vfs_concat.cpp @@ -39,6 +39,41 @@ ConcatenatedVfsFile::ConcatenatedVfsFile(std::map files_, std: ConcatenatedVfsFile::~ConcatenatedVfsFile() = default; +VirtualFile ConcatenatedVfsFile::MakeConcatenatedFile(std::vector files, + std::string name) { + if (files.empty()) + return nullptr; + if (files.size() == 1) + return files[0]; + + return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), std::move(name))); +} + +VirtualFile ConcatenatedVfsFile::MakeConcatenatedFile(u8 filler_byte, + std::map files, + std::string name) { + if (files.empty()) + return nullptr; + if (files.size() == 1) + return files.begin()->second; + + const auto last_valid = --files.end(); + for (auto iter = files.begin(); iter != last_valid;) { + const auto old = iter++; + if (old->first + old->second->GetSize() != iter->first) { + files.emplace(old->first + old->second->GetSize(), + std::make_shared(filler_byte, iter->first - old->first - + old->second->GetSize())); + } + } + + // Ensure the map starts at offset 0 (start of file), otherwise pad to fill. + if (files.begin()->first != 0) + files.emplace(0, std::make_shared(filler_byte, files.begin()->first)); + + return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), std::move(name))); +} + std::string ConcatenatedVfsFile::GetName() const { if (files.empty()) return ""; @@ -101,36 +136,4 @@ bool ConcatenatedVfsFile::Rename(std::string_view name) { return false; } -VirtualFile ConcatenateFiles(std::vector files, std::string name) { - if (files.empty()) - return nullptr; - if (files.size() == 1) - return files[0]; - - return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), std::move(name))); -} - -VirtualFile ConcatenateFiles(u8 filler_byte, std::map files, std::string name) { - if (files.empty()) - return nullptr; - if (files.size() == 1) - return files.begin()->second; - - const auto last_valid = --files.end(); - for (auto iter = files.begin(); iter != last_valid;) { - const auto old = iter++; - if (old->first + old->second->GetSize() != iter->first) { - files.emplace(old->first + old->second->GetSize(), - std::make_shared(filler_byte, iter->first - old->first - - old->second->GetSize())); - } - } - - // Ensure the map starts at offset 0 (start of file), otherwise pad to fill. - if (files.begin()->first != 0) - files.emplace(0, std::make_shared(filler_byte, files.begin()->first)); - - return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), std::move(name))); -} - } // namespace FileSys diff --git a/src/core/file_sys/vfs_concat.h b/src/core/file_sys/vfs_concat.h index 17fa40adee..c90f9d5d1d 100644 --- a/src/core/file_sys/vfs_concat.h +++ b/src/core/file_sys/vfs_concat.h @@ -14,16 +14,20 @@ namespace FileSys { // Class that wraps multiple vfs files and concatenates them, making reads seamless. Currently // read-only. class ConcatenatedVfsFile : public VfsFile { - friend VirtualFile ConcatenateFiles(std::vector files, std::string name); - friend VirtualFile ConcatenateFiles(u8 filler_byte, std::map files, - std::string name); - ConcatenatedVfsFile(std::vector files, std::string name); ConcatenatedVfsFile(std::map files, std::string name); public: ~ConcatenatedVfsFile() override; + /// Wrapper function to allow for more efficient handling of files.size() == 0, 1 cases. + static VirtualFile MakeConcatenatedFile(std::vector files, std::string name); + + /// Convenience function that turns a map of offsets to files into a concatenated file, filling + /// gaps with a given filler byte. + static VirtualFile MakeConcatenatedFile(u8 filler_byte, std::map files, + std::string name); + std::string GetName() const override; std::size_t GetSize() const override; bool Resize(std::size_t new_size) override; @@ -40,11 +44,4 @@ private: std::string name; }; -// Wrapper function to allow for more efficient handling of files.size() == 0, 1 cases. -VirtualFile ConcatenateFiles(std::vector files, std::string name); - -// Convenience function that turns a map of offsets to files into a concatenated file, filling gaps -// with a given filler byte. -VirtualFile ConcatenateFiles(u8 filler_byte, std::map files, std::string name); - } // namespace FileSys diff --git a/src/core/file_sys/vfs_layered.cpp b/src/core/file_sys/vfs_layered.cpp index 45563d7ae3..bfee017256 100644 --- a/src/core/file_sys/vfs_layered.cpp +++ b/src/core/file_sys/vfs_layered.cpp @@ -8,7 +8,13 @@ namespace FileSys { -VirtualDir LayerDirectories(std::vector dirs, std::string name) { +LayeredVfsDirectory::LayeredVfsDirectory(std::vector dirs, std::string name) + : dirs(std::move(dirs)), name(std::move(name)) {} + +LayeredVfsDirectory::~LayeredVfsDirectory() = default; + +VirtualDir LayeredVfsDirectory::MakeLayeredDirectory(std::vector dirs, + std::string name) { if (dirs.empty()) return nullptr; if (dirs.size() == 1) @@ -17,11 +23,6 @@ VirtualDir LayerDirectories(std::vector dirs, std::string name) { return std::shared_ptr(new LayeredVfsDirectory(std::move(dirs), std::move(name))); } -LayeredVfsDirectory::LayeredVfsDirectory(std::vector dirs, std::string name) - : dirs(std::move(dirs)), name(std::move(name)) {} - -LayeredVfsDirectory::~LayeredVfsDirectory() = default; - std::shared_ptr LayeredVfsDirectory::GetFileRelative(std::string_view path) const { for (const auto& layer : dirs) { const auto file = layer->GetFileRelative(path); @@ -41,7 +42,7 @@ std::shared_ptr LayeredVfsDirectory::GetDirectoryRelative( out.push_back(std::move(dir)); } - return LayerDirectories(std::move(out)); + return MakeLayeredDirectory(std::move(out)); } std::shared_ptr LayeredVfsDirectory::GetFile(std::string_view name) const { diff --git a/src/core/file_sys/vfs_layered.h b/src/core/file_sys/vfs_layered.h index 4f6e341abe..d85310f57c 100644 --- a/src/core/file_sys/vfs_layered.h +++ b/src/core/file_sys/vfs_layered.h @@ -9,20 +9,18 @@ namespace FileSys { -// Wrapper function to allow for more efficient handling of dirs.size() == 0, 1 cases. -VirtualDir LayerDirectories(std::vector dirs, std::string name = ""); - // Class that stacks multiple VfsDirectories on top of each other, attempting to read from the first // one and falling back to the one after. The highest priority directory (overwrites all others) // should be element 0 in the dirs vector. class LayeredVfsDirectory : public VfsDirectory { - friend VirtualDir LayerDirectories(std::vector dirs, std::string name); - LayeredVfsDirectory(std::vector dirs, std::string name); public: ~LayeredVfsDirectory() override; + /// Wrapper function to allow for more efficient handling of dirs.size() == 0, 1 cases. + static VirtualDir MakeLayeredDirectory(std::vector dirs, std::string name = ""); + std::shared_ptr GetFileRelative(std::string_view path) const override; std::shared_ptr GetDirectoryRelative(std::string_view path) const override; std::shared_ptr GetFile(std::string_view name) const override; From 57616f9758a23bbb9d1f7c5797c2831926004e49 Mon Sep 17 00:00:00 2001 From: Lioncash Date: Tue, 25 Sep 2018 17:56:14 -0400 Subject: [PATCH 03/11] vfs/etc: Append std:: to size_t usages Given we just recently had a patch backport this from citra, let's try and keep the convention uniform. --- src/core/file_sys/fsmitm_romfsbuild.cpp | 2 +- src/core/file_sys/vfs.cpp | 6 +++--- src/core/file_sys/vfs.h | 9 +++++---- src/core/file_sys/vfs_static.h | 16 ++++++++-------- src/core/file_sys/vfs_vector.cpp | 4 ++-- src/core/file_sys/vfs_vector.h | 8 ++++---- src/yuzu/main.cpp | 14 +++++++------- 7 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/core/file_sys/fsmitm_romfsbuild.cpp b/src/core/file_sys/fsmitm_romfsbuild.cpp index 21fc3d796a..16172445a6 100644 --- a/src/core/file_sys/fsmitm_romfsbuild.cpp +++ b/src/core/file_sys/fsmitm_romfsbuild.cpp @@ -97,7 +97,7 @@ struct RomFSBuildFileContext { RomFSBuildFileContext() : path(""), cur_path_ofs(0), path_len(0) {} }; -static u32 romfs_calc_path_hash(u32 parent, std::string path, u32 start, size_t path_len) { +static u32 romfs_calc_path_hash(u32 parent, std::string path, u32 start, std::size_t path_len) { u32 hash = parent ^ 123456789; for (u32 i = 0; i < path_len; i++) { hash = (hash >> 5) | (hash << 27); diff --git a/src/core/file_sys/vfs.cpp b/src/core/file_sys/vfs.cpp index 5fbea17391..bfe50da73d 100644 --- a/src/core/file_sys/vfs.cpp +++ b/src/core/file_sys/vfs.cpp @@ -463,14 +463,14 @@ bool DeepEquals(const VirtualFile& file1, const VirtualFile& file2, std::size_t return true; } -bool VfsRawCopy(const VirtualFile& src, const VirtualFile& dest, size_t block_size) { +bool VfsRawCopy(const VirtualFile& src, const VirtualFile& dest, std::size_t block_size) { if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable()) return false; if (!dest->Resize(src->GetSize())) return false; std::vector temp(std::min(block_size, src->GetSize())); - for (size_t i = 0; i < src->GetSize(); i += block_size) { + for (std::size_t i = 0; i < src->GetSize(); i += block_size) { const auto read = std::min(block_size, src->GetSize() - i); const auto block = src->Read(temp.data(), read, i); @@ -481,7 +481,7 @@ bool VfsRawCopy(const VirtualFile& src, const VirtualFile& dest, size_t block_si return true; } -bool VfsRawCopyD(const VirtualDir& src, const VirtualDir& dest, size_t block_size) { +bool VfsRawCopyD(const VirtualDir& src, const VirtualDir& dest, std::size_t block_size) { if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable()) return false; diff --git a/src/core/file_sys/vfs.h b/src/core/file_sys/vfs.h index cea4aa8b81..2702916317 100644 --- a/src/core/file_sys/vfs.h +++ b/src/core/file_sys/vfs.h @@ -315,18 +315,19 @@ public: bool Rename(std::string_view name) override; }; -// Compare the two files, byte-for-byte, in increments specificed by block_size -bool DeepEquals(const VirtualFile& file1, const VirtualFile& file2, size_t block_size = 0x1000); +// Compare the two files, byte-for-byte, in increments specified by block_size +bool DeepEquals(const VirtualFile& file1, const VirtualFile& file2, + std::size_t block_size = 0x1000); // A method that copies the raw data between two different implementations of VirtualFile. If you // are using the same implementation, it is probably better to use the Copy method in the parent // directory of src/dest. -bool VfsRawCopy(const VirtualFile& src, const VirtualFile& dest, size_t block_size = 0x1000); +bool VfsRawCopy(const VirtualFile& src, const VirtualFile& dest, std::size_t block_size = 0x1000); // A method that performs a similar function to VfsRawCopy above, but instead copies entire // directories. It suffers the same performance penalties as above and an implementation-specific // Copy should always be preferred. -bool VfsRawCopyD(const VirtualDir& src, const VirtualDir& dest, size_t block_size = 0x1000); +bool VfsRawCopyD(const VirtualDir& src, const VirtualDir& dest, std::size_t block_size = 0x1000); // Checks if the directory at path relative to rel exists. If it does, returns that. If it does not // it attempts to create it and returns the new dir or nullptr on failure. diff --git a/src/core/file_sys/vfs_static.h b/src/core/file_sys/vfs_static.h index 8ad77d3008..44fab51d12 100644 --- a/src/core/file_sys/vfs_static.h +++ b/src/core/file_sys/vfs_static.h @@ -14,7 +14,7 @@ namespace FileSys { class StaticVfsFile : public VfsFile { public: - explicit StaticVfsFile(u8 value, size_t size = 0, std::string name = "", + explicit StaticVfsFile(u8 value, std::size_t size = 0, std::string name = "", VirtualDir parent = nullptr) : value{value}, size{size}, name{std::move(name)}, parent{std::move(parent)} {} @@ -22,11 +22,11 @@ public: return name; } - size_t GetSize() const override { + std::size_t GetSize() const override { return size; } - bool Resize(size_t new_size) override { + bool Resize(std::size_t new_size) override { size = new_size; return true; } @@ -43,23 +43,23 @@ public: return true; } - size_t Read(u8* data, size_t length, size_t offset) const override { + std::size_t Read(u8* data, std::size_t length, std::size_t offset) const override { const auto read = std::min(length, size - offset); std::fill(data, data + read, value); return read; } - size_t Write(const u8* data, size_t length, size_t offset) override { + std::size_t Write(const u8* data, std::size_t length, std::size_t offset) override { return 0; } - boost::optional ReadByte(size_t offset) const override { + boost::optional ReadByte(std::size_t offset) const override { if (offset < size) return value; return boost::none; } - std::vector ReadBytes(size_t length, size_t offset) const override { + std::vector ReadBytes(std::size_t length, std::size_t offset) const override { const auto read = std::min(length, size - offset); return std::vector(read, value); } @@ -71,7 +71,7 @@ public: private: u8 value; - size_t size; + std::size_t size; std::string name; VirtualDir parent; }; diff --git a/src/core/file_sys/vfs_vector.cpp b/src/core/file_sys/vfs_vector.cpp index 7033e2c88c..23cff3fb90 100644 --- a/src/core/file_sys/vfs_vector.cpp +++ b/src/core/file_sys/vfs_vector.cpp @@ -38,13 +38,13 @@ bool VectorVfsFile::IsReadable() const { return true; } -size_t VectorVfsFile::Read(u8* data_, size_t length, size_t offset) const { +std::size_t VectorVfsFile::Read(u8* data_, std::size_t length, std::size_t offset) const { const auto read = std::min(length, data.size() - offset); std::memcpy(data_, data.data() + offset, read); return read; } -size_t VectorVfsFile::Write(const u8* data_, size_t length, size_t offset) { +std::size_t VectorVfsFile::Write(const u8* data_, std::size_t length, std::size_t offset) { if (offset + length > data.size()) data.resize(offset + length); const auto write = std::min(length, data.size() - offset); diff --git a/src/core/file_sys/vfs_vector.h b/src/core/file_sys/vfs_vector.h index 115c3ae952..48a414c980 100644 --- a/src/core/file_sys/vfs_vector.h +++ b/src/core/file_sys/vfs_vector.h @@ -16,13 +16,13 @@ public: ~VectorVfsFile() override; std::string GetName() const override; - size_t GetSize() const override; - bool Resize(size_t new_size) override; + std::size_t GetSize() const override; + bool Resize(std::size_t new_size) override; std::shared_ptr GetContainingDirectory() const override; bool IsWritable() const override; bool IsReadable() const override; - size_t Read(u8* data, size_t length, size_t offset) const override; - size_t Write(const u8* data, size_t length, size_t offset) override; + std::size_t Read(u8* data, std::size_t length, std::size_t offset) const override; + std::size_t Write(const u8* data, std::size_t length, std::size_t offset) override; bool Rename(std::string_view name) override; virtual void Assign(std::vector new_data); diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp index dc8b5407d4..1455edc89c 100644 --- a/src/yuzu/main.cpp +++ b/src/yuzu/main.cpp @@ -760,7 +760,7 @@ void GMainWindow::OnGameListDumpRomFS(u64 program_id, const std::string& game_pa const auto path = fmt::format("{}{:016X}/romfs", FileUtil::GetUserPath(FileUtil::UserPath::DumpDir), program_id); - auto failed = [this, &path]() { + const auto failed = [this, &path] { QMessageBox::warning(this, tr("RomFS Extraction Failed!"), tr("There was an error copying the RomFS files or the user " "cancelled the operation.")); @@ -809,9 +809,9 @@ void GMainWindow::OnGameListDumpRomFS(u64 program_id, const std::string& game_pa const auto full = res == "Full"; - const static std::function calculate_entry_size = + static const std::function calculate_entry_size = [](const FileSys::VirtualDir& dir, bool full) { - size_t out = 0; + std::size_t out = 0; for (const auto& subdir : dir->GetSubdirectories()) out += 1 + calculate_entry_size(subdir, full); return out + full ? dir->GetFiles().size() : 0; @@ -822,10 +822,10 @@ void GMainWindow::OnGameListDumpRomFS(u64 program_id, const std::string& game_pa progress.setWindowModality(Qt::WindowModal); progress.setMinimumDuration(100); - const static std::function + static const std::function qt_raw_copy = [](QProgressDialog& dialog, const FileSys::VirtualDir& src, - const FileSys::VirtualDir& dest, size_t block_size, bool full) { + const FileSys::VirtualDir& dest, std::size_t block_size, bool full) { if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable()) return false; if (dialog.wasCanceled()) @@ -931,7 +931,7 @@ void GMainWindow::OnMenuInstallToNAND() { } const auto qt_raw_copy = [this](const FileSys::VirtualFile& src, - const FileSys::VirtualFile& dest, size_t block_size) { + const FileSys::VirtualFile& dest, std::size_t block_size) { if (src == nullptr || dest == nullptr) return false; if (!dest->Resize(src->GetSize())) From cbb146069a7b20d5687d5d163fd1400af3151485 Mon Sep 17 00:00:00 2001 From: Lioncash Date: Tue, 25 Sep 2018 18:04:22 -0400 Subject: [PATCH 04/11] yuzu/main: Move functions stored into static std::function instances out of OnGameListDumpRomFS() This can cause warnings about static constructors, and is also not ideal performance-wise due to the indirection through std::function. This also keeps the behavior itself separate from the surrounding code, which can make it nicer to read, due to the size of the code. --- src/yuzu/main.cpp | 84 +++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp index 1455edc89c..1b125cbd39 100644 --- a/src/yuzu/main.cpp +++ b/src/yuzu/main.cpp @@ -756,6 +756,46 @@ void GMainWindow::OnGameListOpenFolder(u64 program_id, GameListOpenTarget target QDesktopServices::openUrl(QUrl::fromLocalFile(qpath)); } +static std::size_t CalculateRomFSEntrySize(const FileSys::VirtualDir& dir, bool full) { + std::size_t out = 0; + + for (const auto& subdir : dir->GetSubdirectories()) { + out += 1 + CalculateRomFSEntrySize(subdir, full); + } + + return out + full ? dir->GetFiles().size() : 0; +} + +static bool RomFSRawCopy(QProgressDialog& dialog, const FileSys::VirtualDir& src, + const FileSys::VirtualDir& dest, std::size_t block_size, bool full) { + if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable()) + return false; + if (dialog.wasCanceled()) + return false; + + if (full) { + for (const auto& file : src->GetFiles()) { + const auto out = VfsDirectoryCreateFileWrapper(dest, file->GetName()); + if (!FileSys::VfsRawCopy(file, out, block_size)) + return false; + dialog.setValue(dialog.value() + 1); + if (dialog.wasCanceled()) + return false; + } + } + + for (const auto& dir : src->GetSubdirectories()) { + const auto out = dest->CreateSubdirectory(dir->GetName()); + if (!RomFSRawCopy(dialog, dir, out, block_size, full)) + return false; + dialog.setValue(dialog.value() + 1); + if (dialog.wasCanceled()) + return false; + } + + return true; +} + void GMainWindow::OnGameListDumpRomFS(u64 program_id, const std::string& game_path) { const auto path = fmt::format("{}{:016X}/romfs", FileUtil::GetUserPath(FileUtil::UserPath::DumpDir), program_id); @@ -808,53 +848,13 @@ void GMainWindow::OnGameListDumpRomFS(u64 program_id, const std::string& game_pa failed(); const auto full = res == "Full"; - - static const std::function calculate_entry_size = - [](const FileSys::VirtualDir& dir, bool full) { - std::size_t out = 0; - for (const auto& subdir : dir->GetSubdirectories()) - out += 1 + calculate_entry_size(subdir, full); - return out + full ? dir->GetFiles().size() : 0; - }; - const auto entry_size = calculate_entry_size(extracted, full); + const auto entry_size = CalculateRomFSEntrySize(extracted, full); QProgressDialog progress(tr("Extracting RomFS..."), tr("Cancel"), 0, entry_size, this); progress.setWindowModality(Qt::WindowModal); progress.setMinimumDuration(100); - static const std::function - qt_raw_copy = [](QProgressDialog& dialog, const FileSys::VirtualDir& src, - const FileSys::VirtualDir& dest, std::size_t block_size, bool full) { - if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable()) - return false; - if (dialog.wasCanceled()) - return false; - - if (full) { - for (const auto& file : src->GetFiles()) { - const auto out = VfsDirectoryCreateFileWrapper(dest, file->GetName()); - if (!FileSys::VfsRawCopy(file, out, block_size)) - return false; - dialog.setValue(dialog.value() + 1); - if (dialog.wasCanceled()) - return false; - } - } - - for (const auto& dir : src->GetSubdirectories()) { - const auto out = dest->CreateSubdirectory(dir->GetName()); - if (!qt_raw_copy(dialog, dir, out, block_size, full)) - return false; - dialog.setValue(dialog.value() + 1); - if (dialog.wasCanceled()) - return false; - } - - return true; - }; - - if (qt_raw_copy(progress, extracted, out, 0x400000, full)) { + if (RomFSRawCopy(progress, extracted, out, 0x400000, full)) { progress.close(); QMessageBox::information(this, tr("RomFS Extraction Succeeded!"), tr("The operation completed successfully.")); From f646ca874d8589f4be4a7e6bcce69301e60b24f3 Mon Sep 17 00:00:00 2001 From: Lioncash Date: Tue, 25 Sep 2018 18:06:35 -0400 Subject: [PATCH 05/11] yuzu/main: Resolve precedence bug within CalculateRomFSEntrySize() Ternary operators have a lower precedence than arithmetic operators, so what was actually occurring here is "return (out + full) ? x : y" which most definitely isn't intended, given we calculate out recursively above. We were essentially doing a lot of work for nothing. --- src/yuzu/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp index 1b125cbd39..d74489935a 100644 --- a/src/yuzu/main.cpp +++ b/src/yuzu/main.cpp @@ -763,7 +763,7 @@ static std::size_t CalculateRomFSEntrySize(const FileSys::VirtualDir& dir, bool out += 1 + CalculateRomFSEntrySize(subdir, full); } - return out + full ? dir->GetFiles().size() : 0; + return out + (full ? dir->GetFiles().size() : 0); } static bool RomFSRawCopy(QProgressDialog& dialog, const FileSys::VirtualDir& src, From fa9e0f9c8bf3c14e424d386ec85019e9498f7f12 Mon Sep 17 00:00:00 2001 From: Lioncash Date: Tue, 25 Sep 2018 18:10:47 -0400 Subject: [PATCH 06/11] fsmitm_romfsbuild: Make auto variable into a std::size_t variable within Build() auto x = 0; auto-deduces x to be an int. This is undesirable when working with unsigned values. It also causes sign conversion warnings. Instead, we can make it a proper unsigned value with the correct width that the following expressions operate on. --- src/core/file_sys/fsmitm_romfsbuild.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/file_sys/fsmitm_romfsbuild.cpp b/src/core/file_sys/fsmitm_romfsbuild.cpp index 16172445a6..a8af5e14f3 100644 --- a/src/core/file_sys/fsmitm_romfsbuild.cpp +++ b/src/core/file_sys/fsmitm_romfsbuild.cpp @@ -355,7 +355,7 @@ std::map RomFSBuildContext::Build() { std::vector metadata(file_hash_table_size + file_table_size + dir_hash_table_size + dir_table_size); - auto index = 0; + std::size_t index = 0; std::memcpy(metadata.data(), dir_hash_table.data(), dir_hash_table.size() * sizeof(u32)); index += dir_hash_table.size() * sizeof(u32); std::memcpy(metadata.data() + index, dir_table.data(), dir_table.size()); From 1f92cbc059aaca13823c71a9279393661b050d34 Mon Sep 17 00:00:00 2001 From: Lioncash Date: Tue, 25 Sep 2018 18:14:29 -0400 Subject: [PATCH 07/11] fsmitm_romfsbuild: Remove unnecessary loops in Build() The std::vector instances are already initially allocated with all entries having these values, there's no need to loop through and fill them with it again when they aren't modified. --- src/core/file_sys/fsmitm_romfsbuild.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/core/file_sys/fsmitm_romfsbuild.cpp b/src/core/file_sys/fsmitm_romfsbuild.cpp index a8af5e14f3..b8807e5cad 100644 --- a/src/core/file_sys/fsmitm_romfsbuild.cpp +++ b/src/core/file_sys/fsmitm_romfsbuild.cpp @@ -233,12 +233,6 @@ std::map RomFSBuildContext::Build() { std::vector dir_table(dir_table_size); std::vector file_table(file_table_size); - // Clear out hash tables. - for (u32 i = 0; i < dir_hash_table_entry_count; i++) - dir_hash_table[i] = ROMFS_ENTRY_EMPTY; - for (u32 i = 0; i < file_hash_table_entry_count; i++) - file_hash_table[i] = ROMFS_ENTRY_EMPTY; - std::shared_ptr cur_file; // Determine file offsets. From 91b56c49280355cf4d1d4283ce05645a85aac1fb Mon Sep 17 00:00:00 2001 From: Lioncash Date: Tue, 25 Sep 2018 18:17:42 -0400 Subject: [PATCH 08/11] fsmitm_romfsbuild: Remove unnecessary constructors and initializers for RomFSBuildFileContext and RomFSBuildDirectoryContext There's no need to duplicate in-class initializers with a constructor initializer list. std::strings also initialize to empty by default. --- src/core/file_sys/fsmitm_romfsbuild.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/core/file_sys/fsmitm_romfsbuild.cpp b/src/core/file_sys/fsmitm_romfsbuild.cpp index b8807e5cad..20dd695708 100644 --- a/src/core/file_sys/fsmitm_romfsbuild.cpp +++ b/src/core/file_sys/fsmitm_romfsbuild.cpp @@ -73,7 +73,7 @@ static_assert(sizeof(RomFSFileEntry) == 0x20, "RomFSFileEntry has incorrect size struct RomFSBuildFileContext; struct RomFSBuildDirectoryContext { - std::string path = ""; + std::string path; u32 cur_path_ofs = 0; u32 path_len = 0; u32 entry_offset = 0; @@ -84,7 +84,7 @@ struct RomFSBuildDirectoryContext { }; struct RomFSBuildFileContext { - std::string path = ""; + std::string path; u32 cur_path_ofs = 0; u32 path_len = 0; u32 entry_offset = 0; @@ -92,9 +92,7 @@ struct RomFSBuildFileContext { u64 size = 0; std::shared_ptr parent; std::shared_ptr sibling; - VirtualFile source = nullptr; - - RomFSBuildFileContext() : path(""), cur_path_ofs(0), path_len(0) {} + VirtualFile source; }; static u32 romfs_calc_path_hash(u32 parent, std::string path, u32 start, std::size_t path_len) { From 4654f896184fa1b97df5920ef775f033c2b2fcbb Mon Sep 17 00:00:00 2001 From: Lioncash Date: Tue, 25 Sep 2018 18:25:10 -0400 Subject: [PATCH 09/11] fsmitm_romfsbuild: Avoid type truncation warnings Cast where explicitly necessary and in other cases we can simply modify the algorithm to accomodate larger data. --- src/core/file_sys/fsmitm_romfsbuild.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/core/file_sys/fsmitm_romfsbuild.cpp b/src/core/file_sys/fsmitm_romfsbuild.cpp index 20dd695708..07b0748177 100644 --- a/src/core/file_sys/fsmitm_romfsbuild.cpp +++ b/src/core/file_sys/fsmitm_romfsbuild.cpp @@ -105,13 +105,16 @@ static u32 romfs_calc_path_hash(u32 parent, std::string path, u32 start, std::si return hash; } -static u32 romfs_get_hash_table_count(u32 num_entries) { +static u64 romfs_get_hash_table_count(u64 num_entries) { if (num_entries < 3) { return 3; - } else if (num_entries < 19) { + } + + if (num_entries < 19) { return num_entries | 1; } - u32 count = num_entries; + + u64 count = num_entries; while (count % 2 == 0 || count % 3 == 0 || count % 5 == 0 || count % 7 == 0 || count % 11 == 0 || count % 13 == 0 || count % 17 == 0) { count++; @@ -137,7 +140,7 @@ void RomFSBuildContext::VisitDirectory(VirtualDir root_romfs, const auto child = std::make_shared(); // Set child's path. child->cur_path_ofs = parent->path_len + 1; - child->path_len = child->cur_path_ofs + kv.first.size(); + child->path_len = child->cur_path_ofs + static_cast(kv.first.size()); child->path = parent->path + "/" + kv.first; // Sanity check on path_len @@ -150,7 +153,7 @@ void RomFSBuildContext::VisitDirectory(VirtualDir root_romfs, const auto child = std::make_shared(); // Set child's path. child->cur_path_ofs = parent->path_len + 1; - child->path_len = child->cur_path_ofs + kv.first.size(); + child->path_len = child->cur_path_ofs + static_cast(kv.first.size()); child->path = parent->path + "/" + kv.first; // Sanity check on path_len @@ -217,8 +220,8 @@ RomFSBuildContext::RomFSBuildContext(VirtualDir base_) : base(std::move(base_)) RomFSBuildContext::~RomFSBuildContext() = default; std::map RomFSBuildContext::Build() { - const auto dir_hash_table_entry_count = romfs_get_hash_table_count(num_dirs); - const auto file_hash_table_entry_count = romfs_get_hash_table_count(num_files); + const u64 dir_hash_table_entry_count = romfs_get_hash_table_count(num_dirs); + const u64 file_hash_table_entry_count = romfs_get_hash_table_count(num_files); dir_hash_table_size = 4 * dir_hash_table_entry_count; file_hash_table_size = 4 * file_hash_table_entry_count; From e3b2ef917076d10b7dab45023b5f15b9c0985400 Mon Sep 17 00:00:00 2001 From: Lioncash Date: Tue, 25 Sep 2018 19:56:46 -0400 Subject: [PATCH 10/11] vfs_vector: Amend initializer list order in VectorVfsFile's constructor initializer list Orders the initializer list members to be in the same order that they would be initialized in. Avoids compiler warnings. --- src/core/file_sys/vfs_vector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/file_sys/vfs_vector.cpp b/src/core/file_sys/vfs_vector.cpp index 23cff3fb90..389c7e0036 100644 --- a/src/core/file_sys/vfs_vector.cpp +++ b/src/core/file_sys/vfs_vector.cpp @@ -9,7 +9,7 @@ namespace FileSys { VectorVfsFile::VectorVfsFile(std::vector initial_data, std::string name, VirtualDir parent) - : data(std::move(initial_data)), name(std::move(name)), parent(std::move(parent)) {} + : data(std::move(initial_data)), parent(std::move(parent)), name(std::move(name)) {} VectorVfsFile::~VectorVfsFile() = default; From 11104b48834b2f8e5c59edc46b794e6ab0bd9c4f Mon Sep 17 00:00:00 2001 From: Lioncash Date: Tue, 25 Sep 2018 20:09:20 -0400 Subject: [PATCH 11/11] patch_manager: Invert conditionals within ApplyLayeredFS() Avoids the need to nest code quite a bit by early-exiting in error cases. --- src/core/file_sys/patch_manager.cpp | 63 +++++++++++++++-------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/src/core/file_sys/patch_manager.cpp b/src/core/file_sys/patch_manager.cpp index 561ad67a70..4b3b5e6654 100644 --- a/src/core/file_sys/patch_manager.cpp +++ b/src/core/file_sys/patch_manager.cpp @@ -70,37 +70,40 @@ VirtualDir PatchManager::PatchExeFS(VirtualDir exefs) const { static void ApplyLayeredFS(VirtualFile& romfs, u64 title_id, ContentRecordType type) { const auto load_dir = Service::FileSystem::GetModificationLoadRoot(title_id); - if (type == ContentRecordType::Program && load_dir != nullptr && load_dir->GetSize() > 0) { - auto extracted = ExtractRomFS(romfs); - - if (extracted != nullptr) { - auto patch_dirs = load_dir->GetSubdirectories(); - std::sort(patch_dirs.begin(), patch_dirs.end(), - [](const VirtualDir& l, const VirtualDir& r) { - return l->GetName() < r->GetName(); - }); - - std::vector layers; - layers.reserve(patch_dirs.size() + 1); - for (const auto& subdir : patch_dirs) { - auto romfs_dir = subdir->GetSubdirectory("romfs"); - if (romfs_dir != nullptr) - layers.push_back(std::move(romfs_dir)); - } - - layers.push_back(std::move(extracted)); - - auto layered = LayeredVfsDirectory::MakeLayeredDirectory(std::move(layers)); - if (layered != nullptr) { - auto packed = CreateRomFS(std::move(layered)); - - if (packed != nullptr) { - LOG_INFO(Loader, " RomFS: LayeredFS patches applied successfully"); - romfs = std::move(packed); - } - } - } + if (type != ContentRecordType::Program || load_dir == nullptr || load_dir->GetSize() <= 0) { + return; } + + auto extracted = ExtractRomFS(romfs); + if (extracted == nullptr) { + return; + } + + auto patch_dirs = load_dir->GetSubdirectories(); + std::sort(patch_dirs.begin(), patch_dirs.end(), + [](const VirtualDir& l, const VirtualDir& r) { return l->GetName() < r->GetName(); }); + + std::vector layers; + layers.reserve(patch_dirs.size() + 1); + for (const auto& subdir : patch_dirs) { + auto romfs_dir = subdir->GetSubdirectory("romfs"); + if (romfs_dir != nullptr) + layers.push_back(std::move(romfs_dir)); + } + layers.push_back(std::move(extracted)); + + auto layered = LayeredVfsDirectory::MakeLayeredDirectory(std::move(layers)); + if (layered == nullptr) { + return; + } + + auto packed = CreateRomFS(std::move(layered)); + if (packed == nullptr) { + return; + } + + LOG_INFO(Loader, " RomFS: LayeredFS patches applied successfully"); + romfs = std::move(packed); } VirtualFile PatchManager::PatchRomFS(VirtualFile romfs, u64 ivfc_offset,