From 2fd2345228f1d83445af3d2df66173b3598fbdb6 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 11 Aug 2020 20:08:14 +0800 Subject: [PATCH 01/15] Migrate polarssl to mbedtls. --- .gitmodules | 6 +++--- deps/libmbedtls | 1 + deps/libpolarssl | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) create mode 160000 deps/libmbedtls delete mode 160000 deps/libpolarssl diff --git a/.gitmodules b/.gitmodules index 64f78b5..6775067 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,6 @@ [submodule "deps/liblz4"] path = deps/liblz4 url = https://github.com/jakcron/liblz4.git -[submodule "deps/libpolarssl"] - path = deps/libpolarssl - url = https://github.com/jakcron/libpolarssl.git [submodule "deps/libfnd"] path = deps/libfnd url = https://github.com/jakcron/libfnd.git @@ -19,3 +16,6 @@ [submodule "deps/libnintendo-hac-hb"] path = deps/libnintendo-hac-hb url = https://github.com/jakcron/libnintendo-hac-hb.git +[submodule "deps/libmbedtls"] + path = deps/libmbedtls + url = https://github.com/jakcron/libmbedtls diff --git a/deps/libmbedtls b/deps/libmbedtls new file mode 160000 index 0000000..aa139a4 --- /dev/null +++ b/deps/libmbedtls @@ -0,0 +1 @@ +Subproject commit aa139a4bd25d6d45438e7239a88e1517e9b38aa9 diff --git a/deps/libpolarssl b/deps/libpolarssl deleted file mode 160000 index 7cf5ebe..0000000 --- a/deps/libpolarssl +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7cf5ebe575b5330b946160bad05c6be917340ae9 From 7874c0d1614bac8569b8a05eae5a9d634a0b4c68 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 11 Aug 2020 20:08:34 +0800 Subject: [PATCH 02/15] Update buildsystem links. --- build/visualstudio/nstool.sln | 2 +- build/visualstudio/nstool/nstool.vcxproj | 2 +- makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/visualstudio/nstool.sln b/build/visualstudio/nstool.sln index 8958846..ad181a1 100644 --- a/build/visualstudio/nstool.sln +++ b/build/visualstudio/nstool.sln @@ -36,7 +36,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libnintendo-pki", "..\..\de {4E578016-34BA-4A1E-B8EC-37A48780B6CA} = {4E578016-34BA-4A1E-B8EC-37A48780B6CA} EndProjectSection EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libpolarssl", "..\..\deps\libpolarssl\build\visualstudio\libpolarssl\libpolarssl.vcxproj", "{7A7C66F3-2B5B-4E23-85D8-2A74FEDAD92C}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libmbedtls", "..\..\deps\libmbedtls\build\visualstudio\libmbedtls\libmbedtls.vcxproj", "{7A7C66F3-2B5B-4E23-85D8-2A74FEDAD92C}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/build/visualstudio/nstool/nstool.vcxproj b/build/visualstudio/nstool/nstool.vcxproj index 3b23999..74c4c40 100644 --- a/build/visualstudio/nstool/nstool.vcxproj +++ b/build/visualstudio/nstool/nstool.vcxproj @@ -137,7 +137,7 @@ {0bef63a0-2801-4563-ab65-1e2fd881c3af} - + {7a7c66f3-2b5b-4e23-85d8-2a74fedad92c} diff --git a/makefile b/makefile index fafc39f..4f4ca61 100644 --- a/makefile +++ b/makefile @@ -31,7 +31,7 @@ PROJECT_SONAME = $(PROJECT_NAME).so.$(PROJECT_SO_VER_MAJOR) PROJECT_SO_FILENAME = $(PROJECT_SONAME).$(PROJECT_SO_VER_MINOR).$(PROJECT_SO_VER_PATCH) # Project Dependencies -PROJECT_DEPEND_LOCAL = nintendo-hac-hb nintendo-hac nintendo-es nintendo-pki fnd polarssl lz4 +PROJECT_DEPEND_LOCAL = nintendo-hac-hb nintendo-hac nintendo-es nintendo-pki fnd mbedtls lz4 PROJECT_DEPEND_EXTERNAL = # Generate compiler flags for including project include path From cd1e589216b43279be18fd2c44571fcd90d190f0 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 11 Aug 2020 20:08:47 +0800 Subject: [PATCH 03/15] Update dependencies. --- deps/libfnd | 2 +- deps/libnintendo-es | 2 +- deps/libnintendo-hac | 2 +- deps/libnintendo-hac-hb | 2 +- deps/libnintendo-pki | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deps/libfnd b/deps/libfnd index 234c81d..cebffd2 160000 --- a/deps/libfnd +++ b/deps/libfnd @@ -1 +1 @@ -Subproject commit 234c81d864e53cd208bb93bce69a1f5ff7e44161 +Subproject commit cebffd221d5f25fc31315eabc4b2c776f4a898ae diff --git a/deps/libnintendo-es b/deps/libnintendo-es index ba3036e..a8d5096 160000 --- a/deps/libnintendo-es +++ b/deps/libnintendo-es @@ -1 +1 @@ -Subproject commit ba3036e08dc0d58662c3fc38a3dbee46371e399a +Subproject commit a8d5096bdbf77b88a30559f7b9039ca9d4f22015 diff --git a/deps/libnintendo-hac b/deps/libnintendo-hac index 9372cb3..b1b57ad 160000 --- a/deps/libnintendo-hac +++ b/deps/libnintendo-hac @@ -1 +1 @@ -Subproject commit 9372cb34d8ffdf4c84cc7a25ec9e675fdaff32c6 +Subproject commit b1b57ad02653c08638dcacbf6566a47ae366b4e1 diff --git a/deps/libnintendo-hac-hb b/deps/libnintendo-hac-hb index e7f773b..2049eda 160000 --- a/deps/libnintendo-hac-hb +++ b/deps/libnintendo-hac-hb @@ -1 +1 @@ -Subproject commit e7f773bf883c6f5db700df32a286aa110849d863 +Subproject commit 2049eda044ca50773fa8a4445bfd3d8f1395f8ad diff --git a/deps/libnintendo-pki b/deps/libnintendo-pki index ca22857..9baf593 160000 --- a/deps/libnintendo-pki +++ b/deps/libnintendo-pki @@ -1 +1 @@ -Subproject commit ca228576350a7ea3922ddc87ca43d68376d16fb2 +Subproject commit 9baf5934ad7ba9201cd6a5860fb7905c11fd0101 From 9088f285d898118d5a9f4bb83ee165af06f5d9eb Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 14 Aug 2020 21:36:15 +0800 Subject: [PATCH 04/15] Add support for CompressedRomFs --- deps/libnintendo-hac | 2 +- src/CompressedArchiveIFile.cpp | 193 +++++++++++++++++++++++++++++++++ src/CompressedArchiveIFile.h | 51 +++++++++ src/RomfsProcess.cpp | 47 ++++++++ 4 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 src/CompressedArchiveIFile.cpp create mode 100644 src/CompressedArchiveIFile.h diff --git a/deps/libnintendo-hac b/deps/libnintendo-hac index b1b57ad..5dd0961 160000 --- a/deps/libnintendo-hac +++ b/deps/libnintendo-hac @@ -1 +1 @@ -Subproject commit b1b57ad02653c08638dcacbf6566a47ae366b4e1 +Subproject commit 5dd09615784de624ee8c14032869d30170b58fec diff --git a/src/CompressedArchiveIFile.cpp b/src/CompressedArchiveIFile.cpp new file mode 100644 index 0000000..02f8982 --- /dev/null +++ b/src/CompressedArchiveIFile.cpp @@ -0,0 +1,193 @@ +#include "CompressedArchiveIFile.h" +#include + +#include + +CompressedArchiveIFile::CompressedArchiveIFile(const fnd::SharedPtr& base_file, size_t compression_meta_offset) : + mFile(base_file), + mCompEntries(), + mLogicalFileSize(0), + mCacheCapacity(nn::hac::compression::kRomfsBlockSize), + mCurrentCacheDataSize(0), + mCache(std::shared_ptr(new byte_t[mCacheCapacity])), + mScratch(std::shared_ptr(new byte_t[mCacheCapacity])) +{ + // determine and check the compression metadata size + size_t compression_meta_size = (*mFile)->size() - compression_meta_offset; + if (compression_meta_size % sizeof(nn::hac::sCompressionEntry)) + { + fnd::Exception(kModuleName, "Invalid compression meta size"); + } + + // import raw metadata + std::shared_ptr entries_raw = std::shared_ptr(new byte_t[compression_meta_size]); + (*mFile)->read(entries_raw.get(), compression_meta_offset, compression_meta_size); + + // process metadata entries + nn::hac::sCompressionEntry* entries = (nn::hac::sCompressionEntry*)entries_raw.get(); + for (size_t idx = 0, num = compression_meta_size / sizeof(nn::hac::sCompressionEntry); idx < num; idx++) + { + if (idx == 0) + { + if (entries[idx].physical_offset.get() != 0x0) + throw fnd::Exception(kModuleName, "Entry 0 had a non-zero physical offset"); + if (entries[idx].virtual_offset.get() != 0x0) + throw fnd::Exception(kModuleName, "Entry 0 had a non-zero virtual offset"); + } + else + { + if (entries[idx].physical_offset.get() != align(entries[idx - 1].physical_offset.get() + entries[idx - 1].physical_size.get(), nn::hac::compression::kRomfsBlockAlign)) + throw fnd::Exception(kModuleName, "Entry was not physically aligned with previous entry"); + if (entries[idx].virtual_offset.get() <= entries[idx - 1].virtual_offset.get()) + throw fnd::Exception(kModuleName, "Entry was not virtually aligned with previous entry"); + + // set previous entry virtual_size = this->virtual_offset - prev->virtual_offset; + mCompEntries[mCompEntries.size() - 1].virtual_size = entries[idx].virtual_offset.get() - mCompEntries[mCompEntries.size() - 1].virtual_offset; + } + + if (entries[idx].physical_size.get() > nn::hac::compression::kRomfsBlockSize) + throw fnd::Exception(kModuleName, "Entry physical size was too large"); + + switch ((nn::hac::compression::CompressionType)entries[idx].compression_type) + { + case (nn::hac::compression::CompressionType::None): + case (nn::hac::compression::CompressionType::Lz4): + break; + default: + throw fnd::Exception(kModuleName, "Unsupported CompressionType"); + } + + mCompEntries.push_back({(nn::hac::compression::CompressionType)entries[idx].compression_type, entries[idx].virtual_offset.get(), 0, entries[idx].physical_offset.get(), entries[idx].physical_size.get()}); + } + + // determine logical file size and final entry size + importEntryDataToCache(mCompEntries.size() - 1); + mCompEntries[mCurrentEntryIndex].virtual_size = mCurrentCacheDataSize; + mLogicalFileSize = mCompEntries[mCurrentEntryIndex].virtual_offset + mCompEntries[mCurrentEntryIndex].virtual_size; + + /* + for (auto itr = mCompEntries.begin(); itr != mCompEntries.end(); itr++) + { + std::cout << "entry " << std::endl; + std::cout << " type: " << (uint32_t)itr->compression_type << std::endl; + std::cout << " phys_addr: 0x" << std::hex << itr->physical_offset << std::endl; + std::cout << " phys_size: 0x" << std::hex << itr->physical_size << std::endl; + std::cout << " virt_addr: 0x" << std::hex << itr->virtual_offset << std::endl; + std::cout << " virt_size: 0x" << std::hex << itr->virtual_size << std::endl; + } + + std::cout << "logical size: 0x" << std::hex << mLogicalFileSize << std::endl; + */ +} + +size_t CompressedArchiveIFile::size() +{ + return mLogicalFileSize; +} + +void CompressedArchiveIFile::seek(size_t offset) +{ + mLogicalOffset = std::min(offset, mLogicalFileSize); +} + +void CompressedArchiveIFile::read(byte_t* out, size_t len) +{ + // limit len to the end of the logical file + len = std::min(len, mLogicalFileSize - mLogicalOffset); + + for (size_t pos = 0, entry_index = getEntryIndexForLogicalOffset(mLogicalOffset); pos < len; entry_index++) + { + importEntryDataToCache(entry_index); + + // write padding if required + if (mCompEntries[entry_index].virtual_size > mCurrentCacheDataSize) + { + memset(mCache.get() + mCurrentCacheDataSize, 0, mCompEntries[entry_index].virtual_size - mCurrentCacheDataSize); + } + + // determine + size_t read_offset = mLogicalOffset - (size_t)mCompEntries[entry_index].virtual_offset; + size_t read_size = std::min(len, (size_t)mCompEntries[entry_index].virtual_size - read_offset); + + memcpy(out + pos, mCache.get() + read_offset, read_size); + + pos += read_size; + mLogicalOffset += read_size; + } +} + +void CompressedArchiveIFile::read(byte_t* out, size_t offset, size_t len) +{ + seek(offset); + read(out, len); +} + +void CompressedArchiveIFile::write(const byte_t* out, size_t len) +{ + throw fnd::Exception(kModuleName, "write() not supported"); +} + +void CompressedArchiveIFile::write(const byte_t* out, size_t offset, size_t len) +{ + throw fnd::Exception(kModuleName, "write() not supported"); +} + +void CompressedArchiveIFile::importEntryDataToCache(size_t entry_index) +{ + // return if entry already imported + if (mCurrentEntryIndex == entry_index && mCurrentCacheDataSize != 0) + return; + + // save index + mCurrentEntryIndex = entry_index; + + // reference entry + CompressionEntry& entry = mCompEntries[mCurrentEntryIndex]; + + if (entry.compression_type == nn::hac::compression::CompressionType::None) + { + (*mFile)->read(mCache.get(), entry.physical_offset, entry.physical_size); + mCurrentCacheDataSize = entry.physical_size; + } + else if (entry.compression_type == nn::hac::compression::CompressionType::Lz4) + { + (*mFile)->read(mScratch.get(), entry.physical_offset, entry.physical_size); + + mCurrentCacheDataSize = 0; + fnd::lz4::decompressData(mScratch.get(), entry.physical_size, mCache.get(), mCacheCapacity, mCurrentCacheDataSize); + + if (mCurrentCacheDataSize == 0) + { + throw fnd::Exception(kModuleName, "Decompression of final block failed"); + } + } +} + +size_t CompressedArchiveIFile::getEntryIndexForLogicalOffset(size_t logical_offset) +{ + // rule out bad offset + if (logical_offset > mLogicalFileSize) + throw fnd::Exception(kModuleName, "illegal logical offset"); + + size_t entry_index = 0; + + // try the current comp entry + if (mCompEntries[mCurrentEntryIndex].virtual_offset <= logical_offset && \ + mCompEntries[mCurrentEntryIndex].virtual_offset + mCompEntries[mCurrentEntryIndex].virtual_size >= logical_offset) + { + entry_index = mCurrentEntryIndex; + } + else + { + for (size_t index = 0; index < mCompEntries.size(); index++) + { + if (mCompEntries[index].virtual_offset <= logical_offset && \ + mCompEntries[index].virtual_offset + mCompEntries[index].virtual_size >= logical_offset) + { + entry_index = index; + } + } + } + + return entry_index; +} \ No newline at end of file diff --git a/src/CompressedArchiveIFile.h b/src/CompressedArchiveIFile.h new file mode 100644 index 0000000..795bb49 --- /dev/null +++ b/src/CompressedArchiveIFile.h @@ -0,0 +1,51 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +class CompressedArchiveIFile : public fnd::IFile +{ +public: + CompressedArchiveIFile(const fnd::SharedPtr& file, size_t compression_meta_offset); + + size_t size(); + void seek(size_t offset); + void read(byte_t* out, size_t len); + void read(byte_t* out, size_t offset, size_t len); + void write(const byte_t* out, size_t len); + void write(const byte_t* out, size_t offset, size_t len); +private: + const std::string kModuleName = "CompressedArchiveIFile"; + std::stringstream mErrorSs; + + struct CompressionEntry + { + nn::hac::compression::CompressionType compression_type; + uint64_t virtual_offset; + uint32_t virtual_size; + uint64_t physical_offset; + uint32_t physical_size; + }; + + // raw data + fnd::SharedPtr mFile; + + // compression metadata + std::vector mCompEntries; + size_t mLogicalFileSize; + size_t mLogicalOffset; + + // cached decompressed entry + size_t mCacheCapacity; // capacity + size_t mCurrentEntryIndex; // index of entry currently associated with the cache + uint32_t mCurrentCacheDataSize; // size of data currently in cache + std::shared_ptr mCache; // where decompressed data resides + std::shared_ptr mScratch; // same size as cache, but is used for storing data pre-compression + + // this will import entry to cache + void importEntryDataToCache(size_t entry_index); + size_t getEntryIndexForLogicalOffset(size_t logical_offset); +}; \ No newline at end of file diff --git a/src/RomfsProcess.cpp b/src/RomfsProcess.cpp index aba23fe..c945725 100644 --- a/src/RomfsProcess.cpp +++ b/src/RomfsProcess.cpp @@ -3,6 +3,7 @@ #include #include #include +#include "CompressedArchiveIFile.h" #include "RomfsProcess.h" RomfsProcess::RomfsProcess() : @@ -262,6 +263,52 @@ void RomfsProcess::resolveRomfs() throw fnd::Exception(kModuleName, "Invalid ROMFS Header"); } + // check for romfs compression + size_t physical_size = (*mFile)->size(); + size_t logical_size = mHdr.sections[nn::hac::romfs::FILE_NODE_TABLE].offset.get() + mHdr.sections[nn::hac::romfs::FILE_NODE_TABLE].size.get(); + + // if logical size is greater than the physical size, check for compression meta footer + if (logical_size > physical_size) + { + // initial and final entries + nn::hac::sCompressionEntry entry[2]; + + (*mFile)->read((byte_t*)&entry[1], physical_size - sizeof(nn::hac::sCompressionEntry), sizeof(nn::hac::sCompressionEntry)); + + // the final compression entry should be for the romfs footer, for which the logical offset is detailed in the romfs header + // the compression is always enabled for non-header compression entries + if (entry[1].virtual_offset.get() != mHdr.sections[nn::hac::romfs::DIR_HASHMAP_TABLE].offset.get() || \ + entry[1].compression_type != (byte_t)nn::hac::compression::CompressionType::Lz4) + { + throw fnd::Exception(kModuleName, "RomFs appears corrupted (bad final compression entry)"); + } + + // the first compression entry follows the physical placement of the final data chunk (specified in the final compression entry) + size_t first_entry_offset = align(entry[1].physical_offset.get() + entry[1].physical_size.get(), nn::hac::compression::kRomfsBlockAlign); + + // quick check to make sure the offset at least before the last entry offset + if (first_entry_offset >= (physical_size - sizeof(nn::hac::sCompressionEntry))) + { + throw fnd::Exception(kModuleName, "RomFs appears corrupted (bad final compression entry)"); + } + + // read the first compression entry + (*mFile)->read((byte_t*)&entry[0], first_entry_offset, sizeof(nn::hac::sCompressionEntry)); + + // validate first compression entry + // this should be the same for all compressed romfs + if (entry[0].virtual_offset.get() != 0x0 || \ + entry[0].physical_offset.get() != 0x0 || \ + entry[0].physical_size.get() != 0x200 || \ + entry[0].compression_type != (byte_t)nn::hac::compression::CompressionType::None) + { + throw fnd::Exception(kModuleName, "RomFs appears corrupted (bad first compression entry)"); + } + + // wrap mFile in a class to transparantly decompress the image. + mFile = new CompressedArchiveIFile(mFile, first_entry_offset); + } + // read directory nodes mDirNodes.alloc(mHdr.sections[nn::hac::romfs::DIR_NODE_TABLE].size.get()); (*mFile)->read(mDirNodes.data(), mHdr.sections[nn::hac::romfs::DIR_NODE_TABLE].offset.get(), mDirNodes.size()); From 82b1424abbf7d2e08c85d442fafd39784450dbfd Mon Sep 17 00:00:00 2001 From: jakcron Date: Sun, 16 Aug 2020 14:11:30 +0800 Subject: [PATCH 05/15] [CompressedRomFs] Support romfs footer being compressed in multiple chunks. --- src/RomfsProcess.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/RomfsProcess.cpp b/src/RomfsProcess.cpp index c945725..f03e0be 100644 --- a/src/RomfsProcess.cpp +++ b/src/RomfsProcess.cpp @@ -273,14 +273,18 @@ void RomfsProcess::resolveRomfs() // initial and final entries nn::hac::sCompressionEntry entry[2]; + // read final compression entry (*mFile)->read((byte_t*)&entry[1], physical_size - sizeof(nn::hac::sCompressionEntry), sizeof(nn::hac::sCompressionEntry)); - // the final compression entry should be for the romfs footer, for which the logical offset is detailed in the romfs header + // the final compression entry should be for the (final part, in the case of metadata > 0x10000) romfs footer, for which the logical offset is detailed in the romfs header // the compression is always enabled for non-header compression entries - if (entry[1].virtual_offset.get() != mHdr.sections[nn::hac::romfs::DIR_HASHMAP_TABLE].offset.get() || \ + uint64_t romfs_metadata_begin_offset = mHdr.sections[nn::hac::romfs::DIR_HASHMAP_TABLE].offset.get(); + uint64_t romfs_metadata_end_offset = mHdr.sections[nn::hac::romfs::FILE_NODE_TABLE].offset.get() + mHdr.sections[nn::hac::romfs::FILE_NODE_TABLE].size.get(); + + if ((entry[1].virtual_offset.get() >= romfs_metadata_begin_offset && entry[1].virtual_offset.get() < romfs_metadata_end_offset) == false || \ entry[1].compression_type != (byte_t)nn::hac::compression::CompressionType::Lz4) { - throw fnd::Exception(kModuleName, "RomFs appears corrupted (bad final compression entry)"); + throw fnd::Exception(kModuleName, "RomFs appears corrupted (bad final compression entry virtual offset/compression type)"); } // the first compression entry follows the physical placement of the final data chunk (specified in the final compression entry) @@ -289,10 +293,10 @@ void RomfsProcess::resolveRomfs() // quick check to make sure the offset at least before the last entry offset if (first_entry_offset >= (physical_size - sizeof(nn::hac::sCompressionEntry))) { - throw fnd::Exception(kModuleName, "RomFs appears corrupted (bad final compression entry)"); + throw fnd::Exception(kModuleName, "RomFs appears corrupted (bad final compression entry physical offset/size)"); } - // read the first compression entry + // read first compression entry (*mFile)->read((byte_t*)&entry[0], first_entry_offset, sizeof(nn::hac::sCompressionEntry)); // validate first compression entry From 3ac398cdda3d4a20a21456b4f5b61f79c968d15d Mon Sep 17 00:00:00 2001 From: jakcron Date: Sun, 16 Aug 2020 14:14:15 +0800 Subject: [PATCH 06/15] Add .vscode dir to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8e47c60..96282e5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ data/* *.a *.so.* .DS_Store +.vscode/* ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. From f3e91092a04d36c475cafeba1350534ed31b81eb Mon Sep 17 00:00:00 2001 From: jakcron Date: Sun, 16 Aug 2020 17:40:36 +0800 Subject: [PATCH 07/15] Update readme to reference CompressedRomFS. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 26112c8..38cf29b 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ General purpose reading/extraction tool for Nintendo Switch file formats. ## Supported File Formats * Meta (.npdm) * PartitionFS (and HashedPartitionFS) (includes raw .nsp) -* RomFS -* GameCard Image (.xci) +* RomFS (and CompressedRomFS) +* NX GameCard Image (.xci) * Nintendo Content Archive (.nca) * Content Metadata (.cnmt) * Nintendo Software Object (.nso) From f1de3152ede9f1cd62a3c939cc8aa4060936698f Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 16 Aug 2020 22:08:47 +0800 Subject: [PATCH 08/15] Update visual studio project files. --- build/visualstudio/nstool/nstool.vcxproj | 2 ++ build/visualstudio/nstool/nstool.vcxproj.filters | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/build/visualstudio/nstool/nstool.vcxproj b/build/visualstudio/nstool/nstool.vcxproj index 74c4c40..f56dfbe 100644 --- a/build/visualstudio/nstool/nstool.vcxproj +++ b/build/visualstudio/nstool/nstool.vcxproj @@ -144,6 +144,7 @@ + @@ -168,6 +169,7 @@ + diff --git a/build/visualstudio/nstool/nstool.vcxproj.filters b/build/visualstudio/nstool/nstool.vcxproj.filters index 4ae16db..82a1d26 100644 --- a/build/visualstudio/nstool/nstool.vcxproj.filters +++ b/build/visualstudio/nstool/nstool.vcxproj.filters @@ -21,6 +21,9 @@ Source Files + + Source Files + Source Files @@ -89,6 +92,9 @@ Header Files + + Header Files + Header Files From e560c85cb441b59b3f81a44b128908c789aceb43 Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 16 Aug 2020 22:08:57 +0800 Subject: [PATCH 09/15] Update dependencies. --- deps/libfnd | 2 +- deps/libnintendo-hac | 2 +- deps/libnintendo-pki | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deps/libfnd b/deps/libfnd index cebffd2..8afdc72 160000 --- a/deps/libfnd +++ b/deps/libfnd @@ -1 +1 @@ -Subproject commit cebffd221d5f25fc31315eabc4b2c776f4a898ae +Subproject commit 8afdc7250a4cdb13e3c135581939af5e53913ca8 diff --git a/deps/libnintendo-hac b/deps/libnintendo-hac index 5dd0961..87f14de 160000 --- a/deps/libnintendo-hac +++ b/deps/libnintendo-hac @@ -1 +1 @@ -Subproject commit 5dd09615784de624ee8c14032869d30170b58fec +Subproject commit 87f14de4b143b66f8291507135048323efdb4b3b diff --git a/deps/libnintendo-pki b/deps/libnintendo-pki index 9baf593..2d30779 160000 --- a/deps/libnintendo-pki +++ b/deps/libnintendo-pki @@ -1 +1 @@ -Subproject commit 9baf5934ad7ba9201cd6a5860fb7905c11fd0101 +Subproject commit 2d307790a2febc74aaf5b605fadc12cdf253bcbe From bf4ec5697610bcee36543bed3b18b4fc830f279c Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 16 Aug 2020 22:09:21 +0800 Subject: [PATCH 10/15] Address Visual Studio (2017) warnings. --- src/CompressedArchiveIFile.cpp | 8 +++++--- src/UserSettings.cpp | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/CompressedArchiveIFile.cpp b/src/CompressedArchiveIFile.cpp index 02f8982..4467bfc 100644 --- a/src/CompressedArchiveIFile.cpp +++ b/src/CompressedArchiveIFile.cpp @@ -42,7 +42,7 @@ CompressedArchiveIFile::CompressedArchiveIFile(const fnd::SharedPtr& throw fnd::Exception(kModuleName, "Entry was not virtually aligned with previous entry"); // set previous entry virtual_size = this->virtual_offset - prev->virtual_offset; - mCompEntries[mCompEntries.size() - 1].virtual_size = entries[idx].virtual_offset.get() - mCompEntries[mCompEntries.size() - 1].virtual_offset; + mCompEntries[mCompEntries.size() - 1].virtual_size = uint32_t(entries[idx].virtual_offset.get() - mCompEntries[mCompEntries.size() - 1].virtual_offset); } if (entries[idx].physical_size.get() > nn::hac::compression::kRomfsBlockSize) @@ -97,6 +97,7 @@ void CompressedArchiveIFile::read(byte_t* out, size_t len) for (size_t pos = 0, entry_index = getEntryIndexForLogicalOffset(mLogicalOffset); pos < len; entry_index++) { + // importing entry into cache (this does nothing if the entry is already imported) importEntryDataToCache(entry_index); // write padding if required @@ -105,12 +106,13 @@ void CompressedArchiveIFile::read(byte_t* out, size_t len) memset(mCache.get() + mCurrentCacheDataSize, 0, mCompEntries[entry_index].virtual_size - mCurrentCacheDataSize); } - // determine + // determine subset of cache to copy out size_t read_offset = mLogicalOffset - (size_t)mCompEntries[entry_index].virtual_offset; size_t read_size = std::min(len, (size_t)mCompEntries[entry_index].virtual_size - read_offset); memcpy(out + pos, mCache.get() + read_offset, read_size); + // update position/logical offset pos += read_size; mLogicalOffset += read_size; } @@ -154,7 +156,7 @@ void CompressedArchiveIFile::importEntryDataToCache(size_t entry_index) (*mFile)->read(mScratch.get(), entry.physical_offset, entry.physical_size); mCurrentCacheDataSize = 0; - fnd::lz4::decompressData(mScratch.get(), entry.physical_size, mCache.get(), mCacheCapacity, mCurrentCacheDataSize); + fnd::lz4::decompressData(mScratch.get(), entry.physical_size, mCache.get(), uint32_t(mCacheCapacity), mCurrentCacheDataSize); if (mCurrentCacheDataSize == 0) { diff --git a/src/UserSettings.cpp b/src/UserSettings.cpp index eb2b772..3356974 100644 --- a/src/UserSettings.cpp +++ b/src/UserSettings.cpp @@ -924,12 +924,12 @@ void UserSettings::dumpKeyConfig() const std::cout << " NCA Keys:" << std::endl; for (size_t i = 0; i < kMasterKeyNum; i++) { - if (mKeyCfg.getContentArchiveHeader0SignKey(rsa2048_key, i) == true) + if (mKeyCfg.getContentArchiveHeader0SignKey(rsa2048_key, byte_t(i)) == true) dumpRsa2048Key(rsa2048_key, "Header0-SignatureKey-" + kKeyIndex[i], 2); } for (size_t i = 0; i < kMasterKeyNum; i++) { - if (mKeyCfg.getAcidSignKey(rsa2048_key, i) == true) + if (mKeyCfg.getAcidSignKey(rsa2048_key, byte_t(i)) == true) dumpRsa2048Key(rsa2048_key, "Acid-SignatureKey-" + kKeyIndex[i], 2); } @@ -938,28 +938,28 @@ void UserSettings::dumpKeyConfig() const for (size_t i = 0; i < kMasterKeyNum; i++) { - if (mKeyCfg.getNcaKeyAreaEncryptionKey(i,0, aes_key) == true) + if (mKeyCfg.getNcaKeyAreaEncryptionKey(byte_t(i), 0, aes_key) == true) dumpAesKey(aes_key, "KeyAreaEncryptionKey-Application-" + kKeyIndex[i], 2); - if (mKeyCfg.getNcaKeyAreaEncryptionKey(i,1, aes_key) == true) + if (mKeyCfg.getNcaKeyAreaEncryptionKey(byte_t(i), 1, aes_key) == true) dumpAesKey(aes_key, "KeyAreaEncryptionKey-Ocean-" + kKeyIndex[i], 2); - if (mKeyCfg.getNcaKeyAreaEncryptionKey(i,2, aes_key) == true) + if (mKeyCfg.getNcaKeyAreaEncryptionKey(byte_t(i), 2, aes_key) == true) dumpAesKey(aes_key, "KeyAreaEncryptionKey-System-" + kKeyIndex[i], 2); } for (size_t i = 0; i < kMasterKeyNum; i++) { - if (mKeyCfg.getNcaKeyAreaEncryptionKeyHw(i,0, aes_key) == true) + if (mKeyCfg.getNcaKeyAreaEncryptionKeyHw(byte_t(i), 0, aes_key) == true) dumpAesKey(aes_key, "KeyAreaEncryptionKeyHw-Application-" + kKeyIndex[i], 2); - if (mKeyCfg.getNcaKeyAreaEncryptionKeyHw(i,1, aes_key) == true) + if (mKeyCfg.getNcaKeyAreaEncryptionKeyHw(byte_t(i), 1, aes_key) == true) dumpAesKey(aes_key, "KeyAreaEncryptionKeyHw-Ocean-" + kKeyIndex[i], 2); - if (mKeyCfg.getNcaKeyAreaEncryptionKeyHw(i,2, aes_key) == true) + if (mKeyCfg.getNcaKeyAreaEncryptionKeyHw(byte_t(i), 2, aes_key) == true) dumpAesKey(aes_key, "KeyAreaEncryptionKeyHw-System-" + kKeyIndex[i], 2); } std::cout << " NRR Keys:" << std::endl; for (size_t i = 0; i < kMasterKeyNum; i++) { - if (mKeyCfg.getNrrCertificateSignKey(rsa2048_key, i) == true) + if (mKeyCfg.getNrrCertificateSignKey(rsa2048_key, byte_t(i)) == true) dumpRsa2048Key(rsa2048_key, "Certificate-SignatureKey-" + kKeyIndex[i], 2); } @@ -975,7 +975,7 @@ void UserSettings::dumpKeyConfig() const std::cout << " Package1 Keys:" << std::endl; for (size_t i = 0; i < kMasterKeyNum; i++) { - if (mKeyCfg.getPkg1Key(i, aes_key) == true) + if (mKeyCfg.getPkg1Key(byte_t(i), aes_key) == true) dumpAesKey(aes_key, "EncryptionKey-" + kKeyIndex[i], 2); } @@ -984,14 +984,14 @@ void UserSettings::dumpKeyConfig() const dumpRsa2048Key(rsa2048_key, "Signature Key", 2); for (size_t i = 0; i < kMasterKeyNum; i++) { - if (mKeyCfg.getPkg2Key(i, aes_key) == true) + if (mKeyCfg.getPkg2Key(byte_t(i), aes_key) == true) dumpAesKey(aes_key, "EncryptionKey-" + kKeyIndex[i], 2); } std::cout << " ETicket Keys:" << std::endl; for (size_t i = 0; i < kMasterKeyNum; i++) { - if (mKeyCfg.getETicketCommonKey(i, aes_key) == true) + if (mKeyCfg.getETicketCommonKey(byte_t(i), aes_key) == true) dumpAesKey(aes_key, "CommonKey-" + kKeyIndex[i], 2); } From 3f3bfd33135262aaa0a055090b8023c16005d242 Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 16 Aug 2020 22:09:29 +0800 Subject: [PATCH 11/15] Bump version --- src/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h b/src/version.h index f518f46..529c4cc 100644 --- a/src/version.h +++ b/src/version.h @@ -2,6 +2,6 @@ #define APP_NAME "NSTool" #define BIN_NAME "nstool" #define VER_MAJOR 1 -#define VER_MINOR 3 +#define VER_MINOR 4 #define VER_PATCH 0 #define AUTHORS "jakcron" \ No newline at end of file From c559eb047430f15e12b78a5e1aa16d0ff848521b Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 16 Aug 2020 22:52:57 +0800 Subject: [PATCH 12/15] Update dependencies --- deps/libfnd | 2 +- deps/libmbedtls | 2 +- deps/libnintendo-es | 2 +- deps/libnintendo-hac | 2 +- deps/libnintendo-hac-hb | 2 +- deps/libnintendo-pki | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deps/libfnd b/deps/libfnd index 8afdc72..19c1683 160000 --- a/deps/libfnd +++ b/deps/libfnd @@ -1 +1 @@ -Subproject commit 8afdc7250a4cdb13e3c135581939af5e53913ca8 +Subproject commit 19c1683060a8b39b737da8505b5e23660ed86282 diff --git a/deps/libmbedtls b/deps/libmbedtls index aa139a4..bc43e5e 160000 --- a/deps/libmbedtls +++ b/deps/libmbedtls @@ -1 +1 @@ -Subproject commit aa139a4bd25d6d45438e7239a88e1517e9b38aa9 +Subproject commit bc43e5e079529455749d81d1a3b77a9574d5ab01 diff --git a/deps/libnintendo-es b/deps/libnintendo-es index a8d5096..9e3f1ea 160000 --- a/deps/libnintendo-es +++ b/deps/libnintendo-es @@ -1 +1 @@ -Subproject commit a8d5096bdbf77b88a30559f7b9039ca9d4f22015 +Subproject commit 9e3f1ea763be033f60b3b2db0b2d6e2aac462a37 diff --git a/deps/libnintendo-hac b/deps/libnintendo-hac index 87f14de..c42ffd6 160000 --- a/deps/libnintendo-hac +++ b/deps/libnintendo-hac @@ -1 +1 @@ -Subproject commit 87f14de4b143b66f8291507135048323efdb4b3b +Subproject commit c42ffd666a635c72ed3f7d508eaabf90e8a6dc78 diff --git a/deps/libnintendo-hac-hb b/deps/libnintendo-hac-hb index 2049eda..818ebda 160000 --- a/deps/libnintendo-hac-hb +++ b/deps/libnintendo-hac-hb @@ -1 +1 @@ -Subproject commit 2049eda044ca50773fa8a4445bfd3d8f1395f8ad +Subproject commit 818ebdaf219e156a7a147bd0215b7f11467cee26 diff --git a/deps/libnintendo-pki b/deps/libnintendo-pki index 2d30779..5097871 160000 --- a/deps/libnintendo-pki +++ b/deps/libnintendo-pki @@ -1 +1 @@ -Subproject commit 2d307790a2febc74aaf5b605fadc12cdf253bcbe +Subproject commit 5097871222f6e2cd07b7b8e8b58551e913eb1c15 From 22ea33f532b6d4fe1896bcc1259c02cdb8c18fda Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 16 Aug 2020 23:05:42 +0800 Subject: [PATCH 13/15] Update NACP process to support latest NACP spec. --- src/NacpProcess.cpp | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/NacpProcess.cpp b/src/NacpProcess.cpp index 78bfd4b..1dbe44e 100644 --- a/src/NacpProcess.cpp +++ b/src/NacpProcess.cpp @@ -489,4 +489,37 @@ void NacpProcess::displayNacp() std::cout << " IsEnabled: " << std::boolalpha << mNacp.getJitConfiguration().is_enabled << std::endl; std::cout << " MemorySize: 0x" << std::hex << std::setw(16) << std::setfill('0') << mNacp.getJitConfiguration().memory_size << std::endl; } + + // PlayReportPermission + if (mNacp.getPlayReportPermission() != nn::hac::nacp::PlayReportPermission::None || _HAS_BIT(mCliOutputMode, OUTPUT_EXTENDED)) + { + std::cout << " PlayReportPermission: " << nn::hac::ApplicationControlPropertyUtil::getPlayReportPermissionAsString(mNacp.getPlayReportPermission()) << std::endl; + } + + // CrashScreenshotForProd + if (mNacp.getCrashScreenshotForProd() != nn::hac::nacp::CrashScreenshotForProd::Deny || _HAS_BIT(mCliOutputMode, OUTPUT_EXTENDED)) + { + std::cout << " CrashScreenshotForProd: " << nn::hac::ApplicationControlPropertyUtil::getCrashScreenshotForProdAsString(mNacp.getCrashScreenshotForProd()) << std::endl; + } + + // CrashScreenshotForDev + if (mNacp.getCrashScreenshotForDev() != nn::hac::nacp::CrashScreenshotForDev::Deny || _HAS_BIT(mCliOutputMode, OUTPUT_EXTENDED)) + { + std::cout << " CrashScreenshotForDev: " << nn::hac::ApplicationControlPropertyUtil::getCrashScreenshotForDevAsString(mNacp.getCrashScreenshotForDev()) << std::endl; + } + + // AccessibleLaunchRequiredVersion + if (mNacp.getAccessibleLaunchRequiredVersionApplicationId().size() > 0) + { + std::cout << " AccessibleLaunchRequiredVersion:" << std::endl; + std::cout << " ApplicationId:" << std::endl; + for (auto itr = mNacp.getAccessibleLaunchRequiredVersionApplicationId().begin(); itr != mNacp.getAccessibleLaunchRequiredVersionApplicationId().end(); itr++) + { + std::cout << " 0x" << std::hex << std::setw(16) << std::setfill('0') << *itr << std::endl; + } + } + else if (_HAS_BIT(mCliOutputMode, OUTPUT_EXTENDED)) + { + std::cout << " AccessibleLaunchRequiredVersion: None" << std::endl; + } } \ No newline at end of file From c86754d78dba89dd60d7add09ee336eb347ea37f Mon Sep 17 00:00:00 2001 From: jakcron Date: Mon, 17 Aug 2020 14:30:50 +0800 Subject: [PATCH 14/15] Update libnintendo-hac to 0.5 stable. --- deps/libnintendo-hac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/libnintendo-hac b/deps/libnintendo-hac index c42ffd6..89c2583 160000 --- a/deps/libnintendo-hac +++ b/deps/libnintendo-hac @@ -1 +1 @@ -Subproject commit c42ffd666a635c72ed3f7d508eaabf90e8a6dc78 +Subproject commit 89c258322f1fd48dbda9c0498be52399a3c04a8c From 7cc8cf8ea6f8aef43b5d5059d734b830c7a8a981 Mon Sep 17 00:00:00 2001 From: jakcron Date: Mon, 17 Aug 2020 14:30:58 +0800 Subject: [PATCH 15/15] Update libnintendo-hac-hb to 0.3 stable. --- deps/libnintendo-hac-hb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/libnintendo-hac-hb b/deps/libnintendo-hac-hb index 818ebda..95fb4d7 160000 --- a/deps/libnintendo-hac-hb +++ b/deps/libnintendo-hac-hb @@ -1 +1 @@ -Subproject commit 818ebdaf219e156a7a147bd0215b7f11467cee26 +Subproject commit 95fb4d7762eb5b395fa5023fd3a9b6f34151505a