Merge pull request #7 from jakcron/romfs-development

Romfs development
This commit is contained in:
Jack 2018-04-25 22:28:16 +08:00 committed by GitHub
commit 8ad7edecda
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 558 additions and 29 deletions

View file

@ -6,6 +6,12 @@ namespace fnd
{
namespace io
{
#ifdef _WIN32
const std::string kPathDivider = "\\";
#else
const std::string kPathDivider = "/";
#endif
size_t getFileSize(const std::string& path);
void readFile(const std::string& path, MemoryBlob& blob);
void readFile(const std::string& path, size_t offset, size_t len, MemoryBlob& blob);
@ -13,6 +19,6 @@ namespace fnd
void writeFile(const std::string& path, const byte_t* data, size_t len);
void makeDirectory(const std::string& path);
void getEnvironVar(std::string& var, const std::string& key);
void makePath(std::string& out, const std::vector<std::string>& elements);
void appendToPath(std::string& base, const std::string& add);
}
}

View file

@ -6,6 +6,7 @@ void fnd::SimpleTextOutput::hxdStyleDump(const byte_t* data, size_t len, size_t
// iterate over blocks
for (size_t i = 0; i < (len / row_len); i++)
{
printf("%08" PRIx64 " | ", i*row_len);
// for block i print each byte
for (size_t j = 0; j < row_len; j++)
{
@ -22,6 +23,32 @@ void fnd::SimpleTextOutput::hxdStyleDump(const byte_t* data, size_t len, size_t
}
printf("\n");
}
if ((len % row_len) > 0)
{
size_t i = (len / row_len);
printf("%08" PRIx64 " | ", i * row_len);
// for block i print each byte
for (size_t j = 0; j < row_len; j++)
{
if (j < (len % row_len))
printf("%02X", data[(i * row_len) + j]);
else
printf(" ");
if (((j+1) % byte_grouping_size) == 0)
{
putchar(' ');
}
}
printf(" ");
for (size_t j = 0; j < row_len; j++)
{
if (j < (len % row_len))
printf("%c", isalnum(data[(i * row_len) + j]) ? data[(i * row_len) + j] : '.');
else
printf(" ");
}
printf("\n");
}
}
void fnd::SimpleTextOutput::hxdStyleDump(const byte_t* data, size_t len)

View file

@ -1,4 +1,5 @@
#include <fnd/io.h>
#include <fnd/StringConv.h>
#include <fnd/SimpleFile.h>
#include <fstream>
#ifdef _WIN32
@ -133,7 +134,8 @@ void io::writeFile(const std::string & path, const byte_t * data, size_t len)
void io::makeDirectory(const std::string& path)
{
#ifdef _WIN32
_mkdir(path.c_str());
std::u16string wpath = fnd::StringConv::ConvertChar8ToChar16(path);
_wmkdir((wchar_t*)wpath.c_str());
#else
mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
#endif
@ -163,20 +165,19 @@ void fnd::io::getEnvironVar(std::string & var, const std::string & key)
#endif
}
void fnd::io::makePath(std::string & out, const std::vector<std::string>& elements)
void fnd::io::appendToPath(std::string& base, const std::string& add)
{
out.clear();
out = "";
for (size_t i = 0; i < elements.size(); i++)
if (add.empty())
return;
if (base.empty())
{
if (i > 0)
{
#ifdef _WIN32
out += "\\";
#else
out += "/";
#endif
}
out += elements[i];
base = add;
}
else
{
if (base[base.length()-1] != io::kPathDivider[0])
base += io::kPathDivider;
base += add;
}
}

View file

@ -15,9 +15,11 @@ namespace nx
DIR_NODE_TABLE,
FILE_HASHMAP_TABLE,
FILE_NODE_TABLE,
SECTION_NUM
};
static const uint64_t kRomfsHeaderAlign = 0x200;
static const uint32_t kInvalidAddr = 0xffffffff;
}
#pragma pack(push,1)
@ -31,5 +33,27 @@ namespace nx
} sections[romfs::SECTION_NUM];
le_uint64_t data_offset;
};
struct sRomfsDirEntry
{
le_uint32_t parent;
le_uint32_t sibling;
le_uint32_t child;
le_uint32_t file;
le_uint32_t hash;
le_uint32_t name_size;
char name[];
};
struct sRomfsFileEntry
{
le_uint32_t parent;
le_uint32_t sibling;
le_uint64_t offset;
le_uint64_t size;
le_uint32_t hash;
le_uint32_t name_size;
char name[];
};
#pragma pack(pop)
}

View file

@ -175,6 +175,7 @@
<ClCompile Include="source\main.cpp" />
<ClCompile Include="source\NpdmProcess.cpp" />
<ClCompile Include="source\PfsProcess.cpp" />
<ClCompile Include="source\RomfsProcess.cpp" />
<ClCompile Include="source\UserSettings.cpp" />
<ClCompile Include="source\XciProcess.cpp" />
</ItemGroup>

View file

@ -56,6 +56,9 @@
<ClCompile Include="source\XciProcess.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="source\RomfsProcess.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<None Include="makefile" />

View file

@ -627,6 +627,12 @@ NpdmProcess::NpdmProcess() :
void NpdmProcess::process()
{
fnd::MemoryBlob scratch;
if (mReader == nullptr)
{
throw fnd::Exception(kModuleName, "No file reader set.");
}
scratch.alloc(mReader->size());
mReader->read(scratch.getBytes(), 0, scratch.getSize());

View file

@ -110,6 +110,11 @@ void PfsProcess::process()
{
fnd::MemoryBlob scratch;
if (mReader == nullptr)
{
throw fnd::Exception(kModuleName, "No file reader set.");
}
// open minimum header to get full header size
scratch.alloc(sizeof(nx::sPfsHeader));
mReader->read(scratch.getBytes(), mOffset, scratch.getSize());

View file

@ -0,0 +1,303 @@
#include "RomfsProcess.h"
#include <fnd/SimpleTextOutput.h>
#include <fnd/SimpleFile.h>
#include <fnd/io.h>
void RomfsProcess::printTab(size_t tab) const
{
for (size_t i = 0; i < tab; i++)
{
printf(" ");
}
}
void RomfsProcess::displayFile(const sFile& file, size_t tab) const
{
printTab(tab);
printf("%s", file.name.c_str());
if (mCliOutputType >= OUTPUT_VERBOSE)
{
printf(" (offset=0x%" PRIx64 ", size=0x%" PRIx64 ")", file.offset, file.size);
}
putchar('\n');
}
void RomfsProcess::displayDir(const sDirectory& dir, size_t tab) const
{
if (dir.name.empty() == false)
{
printTab(tab);
printf("%s\n", dir.name.c_str());
}
for (size_t i = 0; i < dir.dir_list.getSize(); i++)
{
displayDir(dir.dir_list[i], tab+1);
}
for (size_t i = 0; i < dir.file_list.getSize(); i++)
{
displayFile(dir.file_list[i], tab+1);
}
}
void RomfsProcess::displayHeader()
{
printf("[RomFS]\n");
printf(" DirNum: %u\n", mDirNum);
printf(" FileNum: %u\n", mFileNum);
if (mMountName.empty() == false)
printf(" MountPoint: %s%s\n", mMountName.c_str(), mMountName.at(mMountName.length()-1) != '/' ? "/" : "");
}
void RomfsProcess::displayFs()
{
displayDir(mRootDir, 1);
}
void RomfsProcess::extractDir(const std::string& path, const sDirectory& dir)
{
std::string dir_path;
std::string file_path;
// make dir path
fnd::io::appendToPath(dir_path, path);
if (dir.name.empty() == false)
fnd::io::appendToPath(dir_path, dir.name);
//printf("dirpath=[%s]\n", dir_path.c_str());
// make directory
fnd::io::makeDirectory(dir_path);
// allocate memory for file extraction
fnd::MemoryBlob scratch;
scratch.alloc(kFileExportBlockSize);
// extract files
fnd::SimpleFile outFile;
for (size_t i = 0; i < dir.file_list.getSize(); i++)
{
file_path.clear();
fnd::io::appendToPath(file_path, dir_path);
fnd::io::appendToPath(file_path, dir.file_list[i].name);
if (mCliOutputType >= OUTPUT_VERBOSE)
printf("extract=[%s]\n", file_path.c_str());
outFile.open(file_path, outFile.Create);
mReader->seek(mOffset + dir.file_list[i].offset);
for (size_t j = 0; j < (dir.file_list[i].size / kFileExportBlockSize); j++)
{
mReader->read(scratch.getBytes(), kFileExportBlockSize);
outFile.write(scratch.getBytes(), kFileExportBlockSize);
}
if (dir.file_list[i].size % kFileExportBlockSize)
{
mReader->read(scratch.getBytes(), dir.file_list[i].size % kFileExportBlockSize);
outFile.write(scratch.getBytes(), dir.file_list[i].size % kFileExportBlockSize);
}
outFile.close();
}
for (size_t i = 0; i < dir.dir_list.getSize(); i++)
{
extractDir(dir_path, dir.dir_list[i]);
}
}
void RomfsProcess::extractFs()
{
extractDir(mExtractPath, mRootDir);
}
bool RomfsProcess::validateHeaderLayout(const nx::sRomfsHeader* hdr) const
{
bool validLayout = true;
if (hdr->header_size.get() != sizeof(nx::sRomfsHeader))
{
validLayout = false;
}
uint64_t pos = hdr->sections[0].offset.get();
for (size_t i = 0; i < nx::romfs::SECTION_NUM; i++)
{
if (hdr->sections[i].offset.get() != pos)
{
validLayout = false;
}
pos += hdr->sections[i].size.get();
}
return validLayout;
}
void RomfsProcess::importDirectory(uint32_t dir_offset, sDirectory& dir)
{
nx::sRomfsDirEntry* d_node = get_dir_node(dir_offset);
/*
printf("[DIR-NODE]\n");
printf(" parent=%08x\n", d_node->parent.get());
printf(" sibling=%08x\n", d_node->sibling.get());
printf(" child=%08x\n", d_node->child.get());
printf(" file=%08x\n", d_node->file.get());
printf(" hash=%08x\n", d_node->hash.get());
printf(" name_size=%08x\n", d_node->name_size.get());
printf(" name=%s\n", d_node->name);
*/
for (uint32_t file_addr = d_node->file.get(); file_addr != nx::romfs::kInvalidAddr; )
{
nx::sRomfsFileEntry* f_node = get_file_node(file_addr);
/*
printf("[FILE-NODE]\n");
printf(" parent=%08x\n", f_node->parent.get());
printf(" sibling=%08x\n", f_node->sibling.get());
printf(" offset=%08" PRIx64 "\n", f_node->offset.get());
printf(" size=%08" PRIx64 "\n", f_node->size.get());
printf(" hash=%08x\n", f_node->hash.get());
printf(" name_size=%08x\n", f_node->name_size.get());
printf(" name=%s\n", f_node->name);
*/
dir.file_list.addElement({std::string(f_node->name, f_node->name_size.get()), mHdr.data_offset.get() + f_node->offset.get(), f_node->size.get()});
file_addr = f_node->sibling.get();
mFileNum++;
}
for (uint32_t child_addr = d_node->child.get(); child_addr != nx::romfs::kInvalidAddr; )
{
nx::sRomfsDirEntry* c_node = get_dir_node(child_addr);
dir.dir_list.addElement({std::string(c_node->name, c_node->name_size.get())});
importDirectory(child_addr, dir.dir_list.atBack());
child_addr = c_node->sibling.get();
mDirNum++;
}
}
void RomfsProcess::resolveRomfs()
{
// read header
mReader->read((byte_t*)&mHdr, mOffset, sizeof(nx::sRomfsHeader));
// logic check on the header layout
if (validateHeaderLayout(&mHdr) == false)
{
throw fnd::Exception(kModuleName, "Invalid ROMFS Header");
}
// read directory nodes
mDirNodes.alloc(mHdr.sections[nx::romfs::DIR_NODE_TABLE].size.get());
mReader->read(mDirNodes.getBytes(), mOffset + mHdr.sections[nx::romfs::DIR_NODE_TABLE].offset.get(), mDirNodes.getSize());
//printf("[RAW DIR NODES]\n");
//fnd::SimpleTextOutput::hxdStyleDump(mDirNodes.getBytes(), mDirNodes.getSize());
// read file nodes
mFileNodes.alloc(mHdr.sections[nx::romfs::FILE_NODE_TABLE].size.get());
mReader->read(mFileNodes.getBytes(), mOffset + mHdr.sections[nx::romfs::FILE_NODE_TABLE].offset.get(), mFileNodes.getSize());
//printf("[RAW FILE NODES]\n");
//fnd::SimpleTextOutput::hxdStyleDump(mFileNodes.getBytes(), mFileNodes.getSize());
// A logic check on the root directory node
if ( get_dir_node(0)->parent.get() != 0 \
|| get_dir_node(0)->sibling.get() != nx::romfs::kInvalidAddr \
|| get_dir_node(0)->hash.get() != nx::romfs::kInvalidAddr \
|| get_dir_node(0)->name_size.get() != 0)
{
throw fnd::Exception(kModuleName, "Invalid root directory node");
}
// import directory into internal structure
mDirNum = 0;
mFileNum = 0;
importDirectory(0, mRootDir);
}
RomfsProcess::RomfsProcess() :
mReader(nullptr),
mOffset(0),
mKeyset(nullptr),
mCliOutputType(OUTPUT_NORMAL),
mVerify(false),
mExtractPath(),
mExtract(false),
mMountName(),
mListFs(false),
mDirNum(0),
mFileNum(0)
{
mRootDir.name.clear();
mRootDir.dir_list.clear();
mRootDir.file_list.clear();
}
void RomfsProcess::process()
{
if (mReader == nullptr)
{
throw fnd::Exception(kModuleName, "No file reader set.");
}
resolveRomfs();
if (mCliOutputType >= OUTPUT_NORMAL)
displayHeader();
if (mListFs || mCliOutputType >= OUTPUT_VERBOSE)
displayFs();
if (mExtract)
extractFs();
}
void RomfsProcess::setInputFile(fnd::IFile& reader)
{
mReader = &reader;
}
void RomfsProcess::setInputFileOffset(size_t offset)
{
mOffset = offset;
}
void RomfsProcess::setKeyset(const sKeyset* keyset)
{
mKeyset = keyset;
}
void RomfsProcess::setCliOutputMode(CliOutputType type)
{
mCliOutputType = type;
}
void RomfsProcess::setVerifyMode(bool verify)
{
mVerify = verify;
}
void RomfsProcess::setMountPointName(const std::string& mount_name)
{
mMountName = mount_name;
}
void RomfsProcess::setExtractPath(const std::string& path)
{
mExtract = true;
mExtractPath = path;
}
void RomfsProcess::setListFs(bool list_fs)
{
mListFs = list_fs;
}
const RomfsProcess::sDirectory& RomfsProcess::getRootDir() const
{
return mRootDir;
}

View file

@ -0,0 +1,148 @@
#pragma once
#include <string>
#include <fnd/types.h>
#include <fnd/IFile.h>
#include <fnd/MemoryBlob.h>
#include <fnd/List.h>
#include <nx/romfs.h>
#include "nstool.h"
class RomfsProcess
{
public:
struct sDirectory;
struct sFile;
struct sDirectory
{
std::string name;
fnd::List<sDirectory> dir_list;
fnd::List<sFile> file_list;
sDirectory& operator=(const sDirectory& other)
{
name = other.name;
dir_list = other.dir_list;
file_list = other.file_list;
return *this;
}
bool operator==(const sDirectory& other) const
{
return (name == other.name) \
&& (dir_list == other.dir_list) \
&& (file_list == other.file_list);
}
bool operator!=(const sDirectory& other) const
{
return !operator==(other);
}
bool operator==(const std::string& other) const
{
return (name == other);
}
bool operator!=(const std::string& other) const
{
return !operator==(other);
}
};
struct sFile
{
std::string name;
uint64_t offset;
uint64_t size;
sFile& operator=(const sFile& other)
{
name = other.name;
offset = other.offset;
size = other.size;
return *this;
}
bool operator==(const sFile& other) const
{
return (name == other.name) \
&& (offset == other.offset) \
&& (size == other.size);
}
bool operator!=(const sFile& other) const
{
return !operator==(other);
}
bool operator==(const std::string& other) const
{
return (name == other);
}
bool operator!=(const std::string& other) const
{
return !operator==(other);
}
};
RomfsProcess();
void process();
// generic
void setInputFile(fnd::IFile& reader);
void setInputFileOffset(size_t offset);
void setKeyset(const sKeyset* keyset);
void setCliOutputMode(CliOutputType type);
void setVerifyMode(bool verify);
// romfs specific
void setMountPointName(const std::string& mount_name);
void setExtractPath(const std::string& path);
void setListFs(bool list_fs);
const sDirectory& getRootDir() const;
private:
const std::string kModuleName = "RomfsProcess";
static const size_t kFileExportBlockSize = 0x1000000;
fnd::IFile* mReader;
size_t mOffset;
const sKeyset* mKeyset;
CliOutputType mCliOutputType;
bool mVerify;
std::string mExtractPath;
bool mExtract;
std::string mMountName;
bool mListFs;
size_t mDirNum;
size_t mFileNum;
nx::sRomfsHeader mHdr;
fnd::MemoryBlob mDirNodes;
fnd::MemoryBlob mFileNodes;
sDirectory mRootDir;
inline nx::sRomfsDirEntry* get_dir_node(uint32_t offset) { return (nx::sRomfsDirEntry*)(mDirNodes.getBytes() + offset); }
inline nx::sRomfsFileEntry* get_file_node(uint32_t offset) { return (nx::sRomfsFileEntry*)(mFileNodes.getBytes() + offset); }
void printTab(size_t tab) const;
void displayFile(const sFile& file, size_t tab) const;
void displayDir(const sDirectory& dir, size_t tab) const;
void displayHeader();
void displayFs();
void extractFile(const std::string& path, const sFile& file);
void extractDir(const std::string& path, const sDirectory& dir);
void extractFs();
bool validateHeaderLayout(const nx::sRomfsHeader* hdr) const;
void importDirectory(uint32_t dir_offset, sDirectory& dir);
void resolveRomfs();
};

View file

@ -265,13 +265,10 @@ void UserSettings::populateKeyset(sCmdArgs& args)
const std::string kKeysetNameStr[2] = {"prod.keys", "dev.keys"};
const std::string kHomeSwitchDirStr = ".switch";
std::vector<std::string> path_list;
path_list.push_back(home);
path_list.push_back(kHomeSwitchDirStr);
path_list.push_back(kKeysetNameStr[args.devkit_keys.isSet ? *args.devkit_keys : 0]);
std::string keyset_path;
fnd::io::makePath(keyset_path, path_list);
fnd::io::appendToPath(keyset_path, home);
fnd::io::appendToPath(keyset_path, kHomeSwitchDirStr);
fnd::io::appendToPath(keyset_path, kKeysetNameStr[args.devkit_keys.isSet ? *args.devkit_keys : 0]);
try
{
@ -577,7 +574,7 @@ FileType UserSettings::determineFileTypeFromFile(const std::string& path)
else if (_ASSERT_SIZE(sizeof(nx::sPfsHeader)) && std::string(_QUICK_CAST(nx::sPfsHeader, 0)->signature, 4) == nx::pfs::kHashedPfsSig)
file_type = FILE_PARTITIONFS;
// test romfs
else if (_ASSERT_SIZE(sizeof(nx::sRomfsHeader)) && _QUICK_CAST(nx::sRomfsHeader, 0)->header_size.get() == sizeof(nx::sRomfsHeader) && _QUICK_CAST(nx::sRomfsHeader, 0)->header_size.get() == _QUICK_CAST(nx::sRomfsHeader, 0)->sections[0].offset.get())
else if (_ASSERT_SIZE(sizeof(nx::sRomfsHeader)) && _QUICK_CAST(nx::sRomfsHeader, 0)->header_size.get() == sizeof(nx::sRomfsHeader) && _QUICK_CAST(nx::sRomfsHeader, 0)->sections[1].offset.get() == (_QUICK_CAST(nx::sRomfsHeader, 0)->sections[0].offset.get() + _QUICK_CAST(nx::sRomfsHeader, 0)->sections[0].size.get()))
file_type = FILE_ROMFS;
// test nca2
else if (_ASSERT_SIZE(nx::nca::kHeaderSize) && std::string(nca_header->signature, 4) == nx::nca::kNca2Sig)

View file

@ -193,6 +193,11 @@ void XciProcess::process()
{
fnd::MemoryBlob scratch;
if (mReader == nullptr)
{
throw fnd::Exception(kModuleName, "No file reader set.");
}
// read header page
mReader->read((byte_t*)&mHdrPage, mOffset, sizeof(nx::sXciHeaderPage));

View file

@ -3,7 +3,7 @@
#include "UserSettings.h"
#include "XciProcess.h"
#include "PfsProcess.h"
//#include "RomfsProcess.h"
#include "RomfsProcess.h"
//#include "NcaProcess.h"
#include "NpdmProcess.h"
@ -54,17 +54,20 @@ int main(int argc, char** argv)
}
else if (user_set.getFileType() == FILE_ROMFS)
{
/*
RomfsProcess romfs;
romfs.setRomfsPath(user_set.getInputPath());
romfs.setExtractPath(user_set.getFsPath());
romfs.setKeyset(user_set.getKeyset());
romfs.setInputFile(inputFile);
romfs.setKeyset(&user_set.getKeyset());
romfs.setCliOutputMode(user_set.getCliOutputType());
romfs.setVerifyMode(user_set.isVerifyFile());
if (user_set.getFsPath().isSet)
romfs.setExtractPath(user_set.getFsPath().var);
romfs.setListFs(user_set.isListFs());
romfs.process();
*/
}
else if (user_set.getFileType() == FILE_NCA)
{