[nstool] Aligned file extraction block sizes

For optimisation, IFile wrappers and processes that all used a cache have similar or the same cache sizes.

programs/nstool/source/HashTreeWrappedIFile.cpp

@@ -164,7 +164,8 @@ void HashTreeWrappedIFile::initialiseDataLayer(const HashTreeMeta& hdr)
 	mDataBlockSize = hdr.getDataLayer().block_size;
 
 	// allocate scratchpad
-	mScratch.alloc(mDataBlockSize * 0x10);
+	//mScratch.alloc(mDataBlockSize * 0x10);
+	mScratch.alloc(align(kFileExportBlockSize, mDataBlockSize));
 }
 
 void HashTreeWrappedIFile::readData(size_t block_offset, size_t block_num)

programs/nstool/source/HashTreeWrappedIFile.h

@@ -21,6 +21,7 @@ public:
 	void write(const byte_t* out, size_t offset, size_t len);
 private:
 	const std::string kModuleName = "HashTreeWrappedIFile";
+	static const size_t kFileExportBlockSize = 0x1000000;
 	std::stringstream mErrorSs;
 
 	bool mOwnIFile;

programs/nstool/source/PfsProcess.cpp

@@ -3,104 +3,6 @@
 #include "OffsetAdjustedIFile.h"
 #include "PfsProcess.h"
 
-void PfsProcess::displayHeader()
-{
-	printf("[PartitionFS]\n");
-	printf("  Type:        %s\n", mPfs.getFsType() == mPfs.TYPE_PFS0? "PFS0" : "HFS0");
-	printf("  FileNum:     %" PRId64 "\n", mPfs.getFileList().getSize());
-	if (mMountName.empty() == false)	
-		printf("  MountPoint:  %s%s\n", mMountName.c_str(), mMountName.at(mMountName.length()-1) != '/' ? "/" : "");
-}
-
-void PfsProcess::displayFs()
-{	
-	for (size_t i = 0; i < mPfs.getFileList().getSize(); i++)
-	{
-		printf("    %s", mPfs.getFileList()[i].name.c_str());
-		if (mCliOutputType >= OUTPUT_VERBOSE)
-		{
-			if (mPfs.getFsType() == mPfs.TYPE_PFS0)
-				printf(" (offset=0x%" PRIx64 ", size=0x%" PRIx64 ")\n", mPfs.getFileList()[i].offset, mPfs.getFileList()[i].size);
-			else
-				printf(" (offset=0x%" PRIx64 ", size=0x%" PRIx64 ", hash_protected_size=0x%" PRIx64 ")\n", mPfs.getFileList()[i].offset, mPfs.getFileList()[i].size, mPfs.getFileList()[i].hash_protected_size);
-		}
-		else
-		{
-			printf("\n");
-		}
-		
-	}
-}
-
-size_t PfsProcess::determineHeaderSize(const nx::sPfsHeader* hdr)
-{
-	size_t fileEntrySize = 0;
-	if (std::string(hdr->signature, 4) == nx::pfs::kPfsSig)
-		fileEntrySize = sizeof(nx::sPfsFile);
-	else
-		fileEntrySize = sizeof(nx::sHashedPfsFile);
-
-	return sizeof(nx::sPfsHeader) + hdr->file_num.get() * fileEntrySize + hdr->name_table_size.get();
-}
-
-bool PfsProcess::validateHeaderMagic(const nx::sPfsHeader* hdr)
-{
-	return std::string(hdr->signature, 4) == nx::pfs::kPfsSig || std::string(hdr->signature, 4) == nx::pfs::kHashedPfsSig;
-}
-
-void PfsProcess::validateHfs()
-{
-	fnd::MemoryBlob scratch;
-	crypto::sha::sSha256Hash hash;
-	const fnd::List<nx::PfsHeader::sFile>& file = mPfs.getFileList();
-	for (size_t i = 0; i < file.getSize(); i++)
-	{
-		scratch.alloc(file[i].hash_protected_size);
-		mReader->read(scratch.getBytes(), file[i].offset, file[i].hash_protected_size);
-		crypto::sha::Sha256(scratch.getBytes(), scratch.getSize(), hash.bytes);
-		if (hash != file[i].hash)
-		{
-			if (mCliOutputType >= OUTPUT_MINIMAL)
-				printf("[WARNING] HFS0 %s%s%s: FAIL (bad hash)\n", !mMountName.empty()? mMountName.c_str() : "", !mMountName.empty()? "/" : "", file[i].name.c_str());
-	
-		}
-	}
-}
-
-void PfsProcess::extractFs()
-{
-	// allocate scratch memory
-	fnd::MemoryBlob scratch;
-	scratch.alloc(kFileExportBlockSize);
-
-	// make extract dir
-	fnd::io::makeDirectory(mExtractPath);
-
-	fnd::SimpleFile outFile;
-	const fnd::List<nx::PfsHeader::sFile>& file = mPfs.getFileList();
-
-	std::string file_path;
-	for (size_t i = 0; i < file.getSize(); i++)
-	{
-		file_path.clear();
-		fnd::io::appendToPath(file_path, mExtractPath);
-		fnd::io::appendToPath(file_path, file[i].name);
-		outFile.open(file_path, outFile.Create);
-		mReader->seek(file[i].offset);
-		for (size_t j = 0; j < (file[i].size / kFileExportBlockSize); j++)
-		{
-			mReader->read(scratch.getBytes(), kFileExportBlockSize);
-			outFile.write(scratch.getBytes(), kFileExportBlockSize);
-		}
-		if (file[i].size % kFileExportBlockSize)
-		{
-			mReader->read(scratch.getBytes(), file[i].size % kFileExportBlockSize);
-			outFile.write(scratch.getBytes(), file[i].size % kFileExportBlockSize);
-		}		
-		outFile.close();
-	}
-}
-
 PfsProcess::PfsProcess() :
 	mReader(nullptr),
 	mCliOutputType(OUTPUT_NORMAL),
@@ -189,3 +91,106 @@ const nx::PfsHeader& PfsProcess::getPfsHeader() const
 {
 	return mPfs;
 }
+
+void PfsProcess::displayHeader()
+{
+	printf("[PartitionFS]\n");
+	printf("  Type:        %s\n", mPfs.getFsType() == mPfs.TYPE_PFS0? "PFS0" : "HFS0");
+	printf("  FileNum:     %" PRId64 "\n", mPfs.getFileList().getSize());
+	if (mMountName.empty() == false)	
+		printf("  MountPoint:  %s%s\n", mMountName.c_str(), mMountName.at(mMountName.length()-1) != '/' ? "/" : "");
+}
+
+void PfsProcess::displayFs()
+{	
+	for (size_t i = 0; i < mPfs.getFileList().getSize(); i++)
+	{
+		printf("    %s", mPfs.getFileList()[i].name.c_str());
+		if (mCliOutputType >= OUTPUT_VERBOSE)
+		{
+			if (mPfs.getFsType() == mPfs.TYPE_PFS0)
+				printf(" (offset=0x%" PRIx64 ", size=0x%" PRIx64 ")\n", mPfs.getFileList()[i].offset, mPfs.getFileList()[i].size);
+			else
+				printf(" (offset=0x%" PRIx64 ", size=0x%" PRIx64 ", hash_protected_size=0x%" PRIx64 ")\n", mPfs.getFileList()[i].offset, mPfs.getFileList()[i].size, mPfs.getFileList()[i].hash_protected_size);
+		}
+		else
+		{
+			printf("\n");
+		}
+		
+	}
+}
+
+size_t PfsProcess::determineHeaderSize(const nx::sPfsHeader* hdr)
+{
+	size_t fileEntrySize = 0;
+	if (std::string(hdr->signature, 4) == nx::pfs::kPfsSig)
+		fileEntrySize = sizeof(nx::sPfsFile);
+	else
+		fileEntrySize = sizeof(nx::sHashedPfsFile);
+
+	return sizeof(nx::sPfsHeader) + hdr->file_num.get() * fileEntrySize + hdr->name_table_size.get();
+}
+
+bool PfsProcess::validateHeaderMagic(const nx::sPfsHeader* hdr)
+{
+	return std::string(hdr->signature, 4) == nx::pfs::kPfsSig || std::string(hdr->signature, 4) == nx::pfs::kHashedPfsSig;
+}
+
+void PfsProcess::validateHfs()
+{
+	// allocate when validate is invoked
+	mFileExtractBlock.alloc(kFileExportBlockSize);
+
+	crypto::sha::sSha256Hash hash;
+	const fnd::List<nx::PfsHeader::sFile>& file = mPfs.getFileList();
+	for (size_t i = 0; i < file.getSize(); i++)
+	{
+		mFileExtractBlock.alloc(file[i].hash_protected_size);
+		mReader->read(mFileExtractBlock.getBytes(), file[i].offset, file[i].hash_protected_size);
+		crypto::sha::Sha256(mFileExtractBlock.getBytes(), mFileExtractBlock.getSize(), hash.bytes);
+		if (hash != file[i].hash)
+		{
+			if (mCliOutputType >= OUTPUT_MINIMAL)
+				printf("[WARNING] HFS0 %s%s%s: FAIL (bad hash)\n", !mMountName.empty()? mMountName.c_str() : "", !mMountName.empty()? "/" : "", file[i].name.c_str());
+	
+		}
+	}
+}
+
+void PfsProcess::extractFs()
+{
+	// allocate only when extractDir is invoked
+	mFileExtractBlock.alloc(kFileExportBlockSize);
+
+	// make extract dir
+	fnd::io::makeDirectory(mExtractPath);
+
+	fnd::SimpleFile outFile;
+	const fnd::List<nx::PfsHeader::sFile>& file = mPfs.getFileList();
+
+	std::string file_path;
+	for (size_t i = 0; i < file.getSize(); i++)
+	{
+		file_path.clear();
+		fnd::io::appendToPath(file_path, mExtractPath);
+		fnd::io::appendToPath(file_path, file[i].name);
+
+		if (mCliOutputType >= OUTPUT_VERBOSE)
+			printf("extract=[%s]\n", file_path.c_str());
+
+		outFile.open(file_path, outFile.Create);
+		mReader->seek(file[i].offset);
+		for (size_t j = 0; j < (file[i].size / kFileExportBlockSize); j++)
+		{
+			mReader->read(mFileExtractBlock.getBytes(), kFileExportBlockSize);
+			outFile.write(mFileExtractBlock.getBytes(), kFileExportBlockSize);
+		}
+		if (file[i].size % kFileExportBlockSize)
+		{
+			mReader->read(mFileExtractBlock.getBytes(), file[i].size % kFileExportBlockSize);
+			outFile.write(mFileExtractBlock.getBytes(), file[i].size % kFileExportBlockSize);
+		}		
+		outFile.close();
+	}
+}

programs/nstool/source/PfsProcess.h

@@ -39,6 +39,8 @@ private:
 	std::string mMountName;
 	bool mListFs;
 
+	fnd::MemoryBlob mFileExtractBlock;
+
 	nx::PfsHeader mPfs;
 
 	void displayHeader();

programs/nstool/source/RomfsProcess.cpp

@@ -4,6 +4,83 @@
 #include "OffsetAdjustedIFile.h"
 #include "RomfsProcess.h"
 
+RomfsProcess::RomfsProcess() :
+	mReader(nullptr),
+	mCliOutputType(OUTPUT_NORMAL),
+	mVerify(false),
+	mExtractPath(),
+	mExtract(false),
+	mMountName(),
+	mListFs(false),
+	mDirNum(0),
+	mFileNum(0)
+{
+	mRootDir.name.clear();
+	mRootDir.dir_list.clear();
+	mRootDir.file_list.clear();
+}
+
+RomfsProcess::~RomfsProcess()
+{
+	if (mReader != nullptr)
+	{
+		delete mReader;
+	}
+}
+
+void RomfsProcess::process()
+{
+	if (mReader == nullptr)
+	{
+		throw fnd::Exception(kModuleName, "No file reader set.");
+	}
+
+	resolveRomfs();	
+
+	if (mCliOutputType >= OUTPUT_NORMAL)
+		displayHeader();
+	if (mListFs || mCliOutputType >= OUTPUT_VERBOSE)
+		displayFs();
+	if (mExtract)
+		extractFs();
+}
+
+void RomfsProcess::setInputFile(fnd::IFile* file, size_t offset, size_t size)
+{
+	mReader = new OffsetAdjustedIFile(file, offset, size);
+}
+
+void RomfsProcess::setCliOutputMode(CliOutputType type)
+{
+	mCliOutputType = type;
+}
+
+void RomfsProcess::setVerifyMode(bool verify)
+{
+	mVerify = verify;
+}
+
+void RomfsProcess::setMountPointName(const std::string& mount_name)
+{
+	mMountName = mount_name;
+}
+
+void RomfsProcess::setExtractPath(const std::string& path)
+{
+	mExtract = true;
+	mExtractPath = path;
+}
+
+void RomfsProcess::setListFs(bool list_fs)
+{
+	mListFs = list_fs;
+}
+
+const RomfsProcess::sDirectory& RomfsProcess::getRootDir() const
+{
+	return mRootDir;
+}
+
 void RomfsProcess::printTab(size_t tab) const
 {
 	for (size_t i = 0; i < tab; i++)
@@ -72,8 +149,10 @@ void RomfsProcess::extractDir(const std::string& path, const sDirectory& dir)
 
 
 	// allocate memory for file extraction
+#ifdef NSTOOL_ALLOC_UNIQUE_SCRATCH
 	fnd::MemoryBlob scratch;
 	scratch.alloc(kFileExportBlockSize);
+#endif
 
 	// extract files
 	fnd::SimpleFile outFile;
@@ -91,13 +170,13 @@ void RomfsProcess::extractDir(const std::string& path, const sDirectory& dir)
 		mReader->seek(dir.file_list[i].offset);
 		for (size_t j = 0; j < (dir.file_list[i].size / kFileExportBlockSize); j++)
 		{
-			mReader->read(scratch.getBytes(), kFileExportBlockSize);
+			mReader->read(mFileExtractBlock.getBytes(), kFileExportBlockSize);
-			outFile.write(scratch.getBytes(), kFileExportBlockSize);
+			outFile.write(mFileExtractBlock.getBytes(), kFileExportBlockSize);
 		}
 		if (dir.file_list[i].size % kFileExportBlockSize)
 		{
-			mReader->read(scratch.getBytes(), dir.file_list[i].size % kFileExportBlockSize);
+			mReader->read(mFileExtractBlock.getBytes(), dir.file_list[i].size % kFileExportBlockSize);
-			outFile.write(scratch.getBytes(), dir.file_list[i].size % kFileExportBlockSize);
+			outFile.write(mFileExtractBlock.getBytes(), dir.file_list[i].size % kFileExportBlockSize);
 		}		
 		outFile.close();
 	}
@@ -111,6 +190,8 @@ void RomfsProcess::extractDir(const std::string& path, const sDirectory& dir)
 
 void RomfsProcess::extractFs()
 {
+	// allocate only when extractDir is invoked
+	mFileExtractBlock.alloc(kFileExportBlockSize);
 	extractDir(mExtractPath, mRootDir);
 }
 
@@ -220,81 +301,4 @@ void RomfsProcess::resolveRomfs()
 	mDirNum = 0;
 	mFileNum = 0;
 	importDirectory(0, mRootDir);
-}
-
-RomfsProcess::RomfsProcess() :
-	mReader(nullptr),
-	mCliOutputType(OUTPUT_NORMAL),
-	mVerify(false),
-	mExtractPath(),
-	mExtract(false),
-	mMountName(),
-	mListFs(false),
-	mDirNum(0),
-	mFileNum(0)
-{
-	mRootDir.name.clear();
-	mRootDir.dir_list.clear();
-	mRootDir.file_list.clear();
-}
-
-RomfsProcess::~RomfsProcess()
-{
-	if (mReader != nullptr)
-	{
-		delete mReader;
-	}
-}
-
-void RomfsProcess::process()
-{
-	if (mReader == nullptr)
-	{
-		throw fnd::Exception(kModuleName, "No file reader set.");
-	}
-
-	resolveRomfs();	
-
-	if (mCliOutputType >= OUTPUT_NORMAL)
-		displayHeader();
-	if (mListFs || mCliOutputType >= OUTPUT_VERBOSE)
-		displayFs();
-	if (mExtract)
-		extractFs();
-}
-
-void RomfsProcess::setInputFile(fnd::IFile* file, size_t offset, size_t size)
-{
-	mReader = new OffsetAdjustedIFile(file, offset, size);
-}
-
-void RomfsProcess::setCliOutputMode(CliOutputType type)
-{
-	mCliOutputType = type;
-}
-
-void RomfsProcess::setVerifyMode(bool verify)
-{
-	mVerify = verify;
-}
-
-void RomfsProcess::setMountPointName(const std::string& mount_name)
-{
-	mMountName = mount_name;
-}
-
-void RomfsProcess::setExtractPath(const std::string& path)
-{
-	mExtract = true;
-	mExtractPath = path;
-}
-
-void RomfsProcess::setListFs(bool list_fs)
-{
-	mListFs = list_fs;
-}
-
-const RomfsProcess::sDirectory& RomfsProcess::getRootDir() const
-{
-	return mRootDir;
 }

programs/nstool/source/RomfsProcess.h

@@ -107,6 +107,7 @@ public:
 private:
 	const std::string kModuleName = "RomfsProcess";
 	static const size_t kFileExportBlockSize = 0x1000000;
+	//static const size_t kFileExportBlockSize = 0x1000000;
 
 	fnd::IFile* mReader;
 	CliOutputType mCliOutputType;
@@ -117,6 +118,8 @@ private:
 	std::string mMountName;
 	bool mListFs;
 
+	fnd::MemoryBlob mFileExtractBlock;
+
 	size_t mDirNum;
 	size_t mFileNum;
 	nx::sRomfsHeader mHdr;