[downloader.mp] Add exponential backoff to retries

Fixes #347
This commit is contained in:
derrod 2021-10-07 10:46:17 +02:00
parent a7d5e37f82
commit 2d366a7704

View file

@ -19,7 +19,7 @@ from legendary.models.downloading import (
class DLWorker(Process): class DLWorker(Process):
def __init__(self, name, queue, out_queue, shm, max_retries=5, def __init__(self, name, queue, out_queue, shm, max_retries=7,
logging_queue=None, dl_timeout=10): logging_queue=None, dl_timeout=10):
super().__init__(name=name) super().__init__(name=name)
self.q = queue self.q = queue
@ -65,6 +65,12 @@ class DLWorker(Process):
try: try:
while tries < self.max_retries: while tries < self.max_retries:
# retry once immediately, otherwise do exponential backoff
if tries > 1:
sleep_time = 2**(tries+1)
logger.debug(f'Sleeping {sleep_time} seconds before retrying...')
time.sleep(sleep_time)
# print('Downloading', job.url) # print('Downloading', job.url)
logger.debug(f'Downloading {job.url}') logger.debug(f'Downloading {job.url}')