From 9131f32c22ce71ee528fa25327808b9a050c6f97 Mon Sep 17 00:00:00 2001 From: Witold Baryluk Date: Thu, 16 Nov 2023 00:40:44 +0000 Subject: [PATCH] [downloader] Avoid buffer copies in worker (#621) This increases peek download speed from about 850MB/s to 960MB/s on my computer. https://github.com/derrod/legendary/issues/620 --- legendary/downloader/mp/workers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/legendary/downloader/mp/workers.py b/legendary/downloader/mp/workers.py index 27cb0bf..4b63192 100644 --- a/legendary/downloader/mp/workers.py +++ b/legendary/downloader/mp/workers.py @@ -126,11 +126,12 @@ class DLWorker(Process): # decompress stuff try: - size = len(chunk.data) + data = chunk.data + size = len(data) if size > job.shm.size: logger.fatal('Downloaded chunk is longer than SharedMemorySegment!') - self.shm.buf[job.shm.offset:job.shm.offset + size] = bytes(chunk.data) + self.shm.buf[job.shm.offset:job.shm.offset + size] = data del chunk self.o_q.put(DownloaderTaskResult(success=True, size_decompressed=size, size_downloaded=compressed, **job.__dict__)) @@ -271,7 +272,7 @@ class FileWorker(Process): if j.shared_memory: shm_offset = j.shared_memory.offset + j.chunk_offset shm_end = shm_offset + j.chunk_size - current_file.write(self.shm.buf[shm_offset:shm_end].tobytes()) + current_file.write(self.shm.buf[shm_offset:shm_end]) elif j.cache_file: with open(os.path.join(self.cache_path, j.cache_file), 'rb') as f: if j.chunk_offset: