mirrors/legendary
1
0
Fork 0
mirror of https://github.com/derrod/legendary.git synced 2024-12-22 01:45:28 +00:00
Code Issues Projects Releases Wiki Activity

[downloader] Avoid buffer copies in worker (#621)

Browse source 
This increases peek download speed from about 850MB/s to 960MB/s on my computer.

https://github.com/derrod/legendary/issues/620
This commit is contained in:
Witold Baryluk 2023-11-16 00:40:44 +00:00 committed by GitHub
parent 450784283d
commit 9131f32c22
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
legendary/downloader/mp/workers.py
View file

@ -126,11 +126,12 @@ class DLWorker(Process):
# decompress stuff # decompress stuff
try: try:
size = len(chunk.data) data = chunk.data
size = len(data)
if size > job.shm.size: if size > job.shm.size:
logger.fatal('Downloaded chunk is longer than SharedMemorySegment!') logger.fatal('Downloaded chunk is longer than SharedMemorySegment!')
self.shm.buf[job.shm.offset:job.shm.offset + size] = bytes(chunk.data) self.shm.buf[job.shm.offset:job.shm.offset + size] = data
del chunk del chunk
self.o_q.put(DownloaderTaskResult(success=True, size_decompressed=size, self.o_q.put(DownloaderTaskResult(success=True, size_decompressed=size,
size_downloaded=compressed, **job.__dict__)) size_downloaded=compressed, **job.__dict__))
@ -271,7 +272,7 @@ class FileWorker(Process):
if j.shared_memory: if j.shared_memory:
shm_offset = j.shared_memory.offset + j.chunk_offset shm_offset = j.shared_memory.offset + j.chunk_offset
shm_end = shm_offset + j.chunk_size shm_end = shm_offset + j.chunk_size
current_file.write(self.shm.buf[shm_offset:shm_end].tobytes()) current_file.write(self.shm.buf[shm_offset:shm_end])
elif j.cache_file: elif j.cache_file:
with open(os.path.join(self.cache_path, j.cache_file), 'rb') as f: with open(os.path.join(self.cache_path, j.cache_file), 'rb') as f:
if j.chunk_offset: if j.chunk_offset: