Add support for CompressedRomFs

deps/libnintendo-hac

@@ -1 +1 @@
-Subproject commit b1b57ad02653c08638dcacbf6566a47ae366b4e1
+Subproject commit 5dd09615784de624ee8c14032869d30170b58fec

src/CompressedArchiveIFile.cpp

@@ -0,0 +1,193 @@
+#include "CompressedArchiveIFile.h"
+#include <fnd/lz4.h>
+
+#include <iostream>
+
+CompressedArchiveIFile::CompressedArchiveIFile(const fnd::SharedPtr<fnd::IFile>& base_file, size_t compression_meta_offset) :
+	mFile(base_file),
+	mCompEntries(),
+	mLogicalFileSize(0),
+	mCacheCapacity(nn::hac::compression::kRomfsBlockSize),
+	mCurrentCacheDataSize(0),
+	mCache(std::shared_ptr<byte_t>(new byte_t[mCacheCapacity])),
+	mScratch(std::shared_ptr<byte_t>(new byte_t[mCacheCapacity]))
+{
+	// determine and check the compression metadata size
+	size_t compression_meta_size = (*mFile)->size() - compression_meta_offset;
+	if (compression_meta_size % sizeof(nn::hac::sCompressionEntry))
+	{
+		fnd::Exception(kModuleName, "Invalid compression meta size");
+	}
+	
+	// import raw metadata
+	std::shared_ptr<byte_t> entries_raw = std::shared_ptr<byte_t>(new byte_t[compression_meta_size]);
+	(*mFile)->read(entries_raw.get(), compression_meta_offset, compression_meta_size);
+
+	// process metadata entries
+	nn::hac::sCompressionEntry* entries = (nn::hac::sCompressionEntry*)entries_raw.get();
+	for (size_t idx = 0, num = compression_meta_size / sizeof(nn::hac::sCompressionEntry); idx < num; idx++)
+	{
+		if (idx == 0)
+		{
+			if (entries[idx].physical_offset.get() != 0x0)
+				throw fnd::Exception(kModuleName, "Entry 0 had a non-zero physical offset");
+			if (entries[idx].virtual_offset.get() != 0x0)
+				throw fnd::Exception(kModuleName, "Entry 0 had a non-zero virtual offset");
+		}
+		else 
+		{
+			if (entries[idx].physical_offset.get() != align(entries[idx - 1].physical_offset.get() + entries[idx - 1].physical_size.get(), nn::hac::compression::kRomfsBlockAlign))
+				throw fnd::Exception(kModuleName, "Entry was not physically aligned with previous entry");
+			if (entries[idx].virtual_offset.get() <= entries[idx - 1].virtual_offset.get())
+				throw fnd::Exception(kModuleName, "Entry was not virtually aligned with previous entry");
+
+			// set previous entry virtual_size = this->virtual_offset - prev->virtual_offset;
+			mCompEntries[mCompEntries.size() - 1].virtual_size = entries[idx].virtual_offset.get() - mCompEntries[mCompEntries.size() - 1].virtual_offset;
+		}
+
+		if (entries[idx].physical_size.get() > nn::hac::compression::kRomfsBlockSize)
+			throw fnd::Exception(kModuleName, "Entry physical size was too large");
+		
+		switch ((nn::hac::compression::CompressionType)entries[idx].compression_type)
+		{
+			case (nn::hac::compression::CompressionType::None):
+			case (nn::hac::compression::CompressionType::Lz4):
+				break;
+			default:
+				throw fnd::Exception(kModuleName, "Unsupported CompressionType");
+		}
+
+		mCompEntries.push_back({(nn::hac::compression::CompressionType)entries[idx].compression_type, entries[idx].virtual_offset.get(), 0, entries[idx].physical_offset.get(), entries[idx].physical_size.get()});
+	}
+
+	// determine logical file size and final entry size
+	importEntryDataToCache(mCompEntries.size() - 1);
+	mCompEntries[mCurrentEntryIndex].virtual_size = mCurrentCacheDataSize;
+	mLogicalFileSize = mCompEntries[mCurrentEntryIndex].virtual_offset + mCompEntries[mCurrentEntryIndex].virtual_size;
+
+	/*
+	for (auto itr = mCompEntries.begin(); itr != mCompEntries.end(); itr++)
+	{
+		std::cout << "entry " << std::endl;
+		std::cout << "  type:       " << (uint32_t)itr->compression_type << std::endl;
+		std::cout << "  phys_addr:  0x" << std::hex << itr->physical_offset << std::endl;
+		std::cout << "  phys_size:  0x" << std::hex << itr->physical_size << std::endl;
+		std::cout << "  virt_addr:  0x" << std::hex << itr->virtual_offset << std::endl;
+		std::cout << "  virt_size:  0x" << std::hex << itr->virtual_size << std::endl;
+	}
+
+	std::cout << "logical size: 0x" << std::hex << mLogicalFileSize << std::endl;
+	*/
+}
+
+size_t CompressedArchiveIFile::size()
+{
+	return mLogicalFileSize;
+}
+
+void CompressedArchiveIFile::seek(size_t offset)
+{
+	mLogicalOffset = std::min<size_t>(offset, mLogicalFileSize);
+}
+
+void CompressedArchiveIFile::read(byte_t* out, size_t len)
+{
+	// limit len to the end of the logical file
+	len = std::min<size_t>(len, mLogicalFileSize - mLogicalOffset);
+
+	for (size_t pos = 0, entry_index = getEntryIndexForLogicalOffset(mLogicalOffset); pos < len; entry_index++)
+	{
+		importEntryDataToCache(entry_index);
+
+		// write padding if required
+		if (mCompEntries[entry_index].virtual_size > mCurrentCacheDataSize)
+		{
+			memset(mCache.get() + mCurrentCacheDataSize, 0, mCompEntries[entry_index].virtual_size - mCurrentCacheDataSize);
+		}
+		
+		// determine 
+		size_t read_offset = mLogicalOffset - (size_t)mCompEntries[entry_index].virtual_offset;
+		size_t read_size = std::min<size_t>(len, (size_t)mCompEntries[entry_index].virtual_size - read_offset);
+
+		memcpy(out + pos, mCache.get() + read_offset, read_size);
+
+		pos += read_size;
+		mLogicalOffset += read_size;
+	}
+}
+
+void CompressedArchiveIFile::read(byte_t* out, size_t offset, size_t len)
+{
+	seek(offset);
+	read(out, len);
+}
+
+void CompressedArchiveIFile::write(const byte_t* out, size_t len)
+{
+	throw fnd::Exception(kModuleName, "write() not supported");
+}
+
+void CompressedArchiveIFile::write(const byte_t* out, size_t offset, size_t len)
+{
+	throw fnd::Exception(kModuleName, "write() not supported");
+}
+
+void CompressedArchiveIFile::importEntryDataToCache(size_t entry_index)
+{
+	// return if entry already imported
+	if (mCurrentEntryIndex == entry_index && mCurrentCacheDataSize != 0)
+		return;
+
+	// save index
+	mCurrentEntryIndex = entry_index;
+	
+	// reference entry
+	CompressionEntry& entry = mCompEntries[mCurrentEntryIndex]; 
+
+	if (entry.compression_type == nn::hac::compression::CompressionType::None)
+	{
+		(*mFile)->read(mCache.get(), entry.physical_offset, entry.physical_size);
+		mCurrentCacheDataSize = entry.physical_size;
+	}
+	else if (entry.compression_type == nn::hac::compression::CompressionType::Lz4)
+	{
+		(*mFile)->read(mScratch.get(), entry.physical_offset, entry.physical_size);
+
+		mCurrentCacheDataSize = 0;
+		fnd::lz4::decompressData(mScratch.get(), entry.physical_size, mCache.get(), mCacheCapacity, mCurrentCacheDataSize);
+
+		if (mCurrentCacheDataSize == 0)
+		{
+			throw fnd::Exception(kModuleName, "Decompression of final block failed");
+		}
+	}
+}
+
+size_t CompressedArchiveIFile::getEntryIndexForLogicalOffset(size_t logical_offset)
+{
+	// rule out bad offset
+	if (logical_offset > mLogicalFileSize)
+		throw fnd::Exception(kModuleName, "illegal logical offset");
+
+	size_t entry_index = 0;
+
+	// try the current comp entry 
+	if (mCompEntries[mCurrentEntryIndex].virtual_offset <= logical_offset && \
+		mCompEntries[mCurrentEntryIndex].virtual_offset + mCompEntries[mCurrentEntryIndex].virtual_size >= logical_offset)
+	{
+		entry_index = mCurrentEntryIndex;
+	}
+	else
+	{
+		for (size_t index = 0; index < mCompEntries.size(); index++)
+		{
+			if (mCompEntries[index].virtual_offset <= logical_offset && \
+				mCompEntries[index].virtual_offset + mCompEntries[index].virtual_size >= logical_offset)
+			{
+				entry_index = index;
+			}
+		}
+	}
+
+	return entry_index;
+}

src/CompressedArchiveIFile.h

@@ -0,0 +1,51 @@
+#pragma once
+#include <sstream>
+#include <fnd/IFile.h>
+#include <fnd/SharedPtr.h>
+#include <memory>
+#include <vector>
+#include <nn/hac/define/compression.h>
+
+class CompressedArchiveIFile : public fnd::IFile
+{
+public:
+	CompressedArchiveIFile(const fnd::SharedPtr<fnd::IFile>& file, size_t compression_meta_offset);
+
+	size_t size();
+	void seek(size_t offset);
+	void read(byte_t* out, size_t len);
+	void read(byte_t* out, size_t offset, size_t len);
+	void write(const byte_t* out, size_t len);
+	void write(const byte_t* out, size_t offset, size_t len);
+private:
+	const std::string kModuleName = "CompressedArchiveIFile";
+	std::stringstream mErrorSs;
+
+	struct CompressionEntry
+	{
+		nn::hac::compression::CompressionType compression_type;
+		uint64_t virtual_offset;
+		uint32_t virtual_size;
+		uint64_t physical_offset;
+		uint32_t physical_size;		
+	};
+
+	// raw data
+	fnd::SharedPtr<fnd::IFile> mFile;
+
+	// compression metadata
+	std::vector<CompressionEntry> mCompEntries;
+	size_t mLogicalFileSize;
+	size_t mLogicalOffset;
+
+	// cached decompressed entry
+	size_t mCacheCapacity; // capacity 
+	size_t mCurrentEntryIndex; // index of entry currently associated with the cache
+	uint32_t mCurrentCacheDataSize; // size of data currently in cache
+	std::shared_ptr<byte_t> mCache; // where decompressed data resides
+	std::shared_ptr<byte_t> mScratch; // same size as cache, but is used for storing data pre-compression
+
+	// this will import entry to cache
+	void importEntryDataToCache(size_t entry_index);
+	size_t getEntryIndexForLogicalOffset(size_t logical_offset);
+};

src/RomfsProcess.cpp

@@ -3,6 +3,7 @@
 #include <fnd/SimpleTextOutput.h>
 #include <fnd/SimpleFile.h>
 #include <fnd/io.h>
+#include "CompressedArchiveIFile.h"
 #include "RomfsProcess.h"
 
 RomfsProcess::RomfsProcess() :
@@ -262,6 +263,52 @@ void RomfsProcess::resolveRomfs()
 		throw fnd::Exception(kModuleName, "Invalid ROMFS Header");
 	}
 
+	// check for romfs compression
+	size_t physical_size = (*mFile)->size();
+	size_t logical_size = mHdr.sections[nn::hac::romfs::FILE_NODE_TABLE].offset.get() + mHdr.sections[nn::hac::romfs::FILE_NODE_TABLE].size.get();
+	
+	// if logical size is greater than the physical size, check for compression meta footer
+	if (logical_size > physical_size)
+	{
+		// initial and final entries
+		nn::hac::sCompressionEntry entry[2];
+
+		(*mFile)->read((byte_t*)&entry[1], physical_size - sizeof(nn::hac::sCompressionEntry), sizeof(nn::hac::sCompressionEntry));
+		
+		// the final compression entry should be for the romfs footer, for which the logical offset is detailed in the romfs header
+		// the compression is always enabled for non-header compression entries
+		if (entry[1].virtual_offset.get() != mHdr.sections[nn::hac::romfs::DIR_HASHMAP_TABLE].offset.get() || \
+			entry[1].compression_type != (byte_t)nn::hac::compression::CompressionType::Lz4)
+		{
+			throw fnd::Exception(kModuleName, "RomFs appears corrupted (bad final compression entry)");
+		}
+		
+		// the first compression entry follows the physical placement of the final data chunk (specified in the final compression entry)
+		size_t first_entry_offset = align(entry[1].physical_offset.get() + entry[1].physical_size.get(), nn::hac::compression::kRomfsBlockAlign);
+
+		// quick check to make sure the offset at least before the last entry offset
+		if (first_entry_offset >= (physical_size - sizeof(nn::hac::sCompressionEntry)))
+		{
+			throw fnd::Exception(kModuleName, "RomFs appears corrupted (bad final compression entry)");
+		}
+
+		// read the first compression entry
+		(*mFile)->read((byte_t*)&entry[0], first_entry_offset, sizeof(nn::hac::sCompressionEntry));
+
+		// validate first compression entry
+		// this should be the same for all compressed romfs
+		if (entry[0].virtual_offset.get() != 0x0 || \
+			entry[0].physical_offset.get() != 0x0 || \
+			entry[0].physical_size.get() != 0x200 || \
+			entry[0].compression_type != (byte_t)nn::hac::compression::CompressionType::None)
+		{
+			throw fnd::Exception(kModuleName, "RomFs appears corrupted (bad first compression entry)");
+		}
+
+		// wrap mFile in a class to transparantly decompress the image.
+		mFile = new CompressedArchiveIFile(mFile, first_entry_offset);
+	}
+
 	// read directory nodes
 	mDirNodes.alloc(mHdr.sections[nn::hac::romfs::DIR_NODE_TABLE].size.get());
 	(*mFile)->read(mDirNodes.data(), mHdr.sections[nn::hac::romfs::DIR_NODE_TABLE].offset.get(), mDirNodes.size());