From 2d366a77045c5d045aadb4afb2e221914682eb33 Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 7 Oct 2021 10:46:17 +0200 Subject: [PATCH] [downloader.mp] Add exponential backoff to retries Fixes #347 --- legendary/downloader/mp/workers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/legendary/downloader/mp/workers.py b/legendary/downloader/mp/workers.py index 927e86b..74e34e4 100644 --- a/legendary/downloader/mp/workers.py +++ b/legendary/downloader/mp/workers.py @@ -19,7 +19,7 @@ from legendary.models.downloading import ( class DLWorker(Process): - def __init__(self, name, queue, out_queue, shm, max_retries=5, + def __init__(self, name, queue, out_queue, shm, max_retries=7, logging_queue=None, dl_timeout=10): super().__init__(name=name) self.q = queue @@ -65,6 +65,12 @@ class DLWorker(Process): try: while tries < self.max_retries: + # retry once immediately, otherwise do exponential backoff + if tries > 1: + sleep_time = 2**(tries+1) + logger.debug(f'Sleeping {sleep_time} seconds before retrying...') + time.sleep(sleep_time) + # print('Downloading', job.url) logger.debug(f'Downloading {job.url}')