mirrors/legendary
1
0
Fork 0
mirror of https://github.com/derrod/legendary.git synced 2024-12-22 17:55:27 +00:00
Code Issues Projects Releases Wiki Activity

[models/utils] Track file offset for chunk parts

Browse source 
Preparation for smarter patching with downloader rewrite.
This commit is contained in:
derrod 2021-01-10 14:25:35 +01:00
parent a2280edea8
commit edad963200
3 changed files with 16 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
legendary/models/json_manifest.py
View file

@ -157,15 +157,18 @@ class JSONFML(FML):
_fm.chunk_parts = []
_fm.install_tags = _fmj.pop('InstallTags', list())
_offset = 0
for _cpj in _fmj.pop('FileChunkParts'):
_cp = ChunkPart()
_cp.guid = guid_from_json(_cpj.pop('Guid'))
_cp.offset = blob_to_num(_cpj.pop('Offset'))
_cp.size = blob_to_num(_cpj.pop('Size'))
_cp.file_offset = _offset
_fm.file_size += _cp.size
if _cpj:
print(f'Non-read ChunkPart keys: {_cpj.keys()}')
_fm.chunk_parts.append(_cp)
_offset += _cp.size
if _fmj:
print(f'Non-read FileManifest keys: {_fmj.keys()}')

14
legendary/models/manifest.py
View file

@ -507,13 +507,20 @@ class FML:
# Each file is made up of "Chunk Parts" that can be spread across the "chunk stream"
for fm in _fml.elements:
_elem = struct.unpack('<I', bio.read(4))[0]
_offset = 0
for i in range(_elem):
chunkp = ChunkPart()
_start = bio.tell()
_size = struct.unpack('<I', bio.read(4))[0]
chunkp.guid = struct.unpack('<IIII', bio.read(16))
chunkp.offset = struct.unpack('<I', bio.read(4))[0]
chunkp.size = struct.unpack('<I', bio.read(4))[0]
chunkp.file_offset = _offset
fm.chunk_parts.append(chunkp)
_offset += chunkp.size
if (diff := (bio.tell() - _start - _size)) > 0:
logger.warning(f'Did not read {diff} bytes from chunk part!')
bio.seek(diff)
# we have to calculate the actual file size ourselves
for fm in _fml.elements:
@ -601,10 +608,11 @@ class FileManifest:
class ChunkPart:
def __init__(self, guid=None, offset=0, size=0):
def __init__(self, guid=None, offset=0, size=0, file_offset=0):
self.guid = guid
self.offset = offset
self.size = size
self.file_offset = file_offset
# caches for things that are "expensive" to compute
self._guid_str = None
self._guid_num = None
@ -623,8 +631,8 @@ class ChunkPart:
def __repr__(self):
guid_readable = '-'.join('{:08x}'.format(g) for g in self.guid)
return '<ChunkPart (guid={}, offset={}, size={})>'.format(
guid_readable, self.offset, self.size)
return '<ChunkPart (guid={}, offset={}, size={}, file_offset={})>'.format(
guid_readable, self.offset, self.size, self.file_offset)
class CustomFields:

3
legendary/utils/savegame_helper.py
View file

@ -123,7 +123,8 @@ class SaveGameHelper:
# create chunk part and write it to chunk buffer
cp = ChunkPart(guid=cur_chunk.guid, offset=cur_buffer.tell(),
size=min(remaining, 1024 * 1024 - cur_buffer.tell()))
size=min(remaining, 1024 * 1024 - cur_buffer.tell()),
file_offset=cf.tell())
_tmp = cf.read(cp.size)
if not _tmp:
self.log.warning(f'Got EOF for "{f.filename}" with {remaining} bytes remaining! '