[cli/core/downloader] Increase timeout and make it configurable

Fixes #16

legendary/cli.py

@@ -286,7 +286,8 @@ class LegendaryCLI:
                                                           file_prefix_filter=args.file_prefix,
                                                           file_exclude_filter=args.file_exclude_prefix,
                                                           file_install_tag=args.install_tag,
-                                                          dl_optimizations=args.order_opt)
+                                                          dl_optimizations=args.order_opt,
+                                                          dl_timeout=args.dl_timeout)
 
         # game is either up to date or hasn't changed, so we have nothing to do
         if not analysis.dl_size:
@@ -471,6 +472,8 @@ def main():
                                 type=str, help='Only download files with the specified install tag (testing)')
     install_parser.add_argument('--enable-reordering', dest='order_opt', action='store_true',
                                 help='Enable reordering to attempt to optimize RAM usage during download')
+    install_parser.add_argument('--dl-timeout', dest='dl_timeout', action='store', metavar='<sec>', type=int,
+                                help='Connection timeout for downloader (default: 10 seconds)')
 
     launch_parser.add_argument('--offline', dest='offline', action='store_true',
                                default=False, help='Skip login and launch game without online authentication')

legendary/core.py

@@ -363,7 +363,8 @@ class LegendaryCore:
                          override_old_manifest: str = '', override_base_url: str = '',
                          platform_override: str = '', file_prefix_filter: str = '',
                          file_exclude_filter: str = '', file_install_tag: str = '',
-                         dl_optimizations: bool = False) -> (DLManager, AnalysisResult, ManifestMeta):
+                         dl_optimizations: bool = False, dl_timeout: int = 10
+                         ) -> (DLManager, AnalysisResult, ManifestMeta):
         # load old manifest
         old_manifest = None
 
@@ -436,7 +437,8 @@ class LegendaryCore:
             process_opt = False
 
         dlm = DLManager(install_path, base_url, resume_file=resume_file, status_q=status_q,
-                        max_shared_memory=max_shm * 1024 * 1024, max_workers=max_workers)
+                        max_shared_memory=max_shm * 1024 * 1024, max_workers=max_workers,
+                        dl_timeout=dl_timeout)
         anlres = dlm.run_analysis(manifest=new_manifest, old_manifest=old_manifest,
                                   patch=not disable_patching, resume=not force,
                                   file_prefix_filter=file_prefix_filter,

legendary/downloader/manager.py

@@ -23,7 +23,7 @@ from legendary.models.manifest import ManifestComparison, Manifest
 class DLManager(Process):
     def __init__(self, download_dir, base_url, cache_dir=None, status_q=None,
                  max_jobs=100, max_failures=5, max_workers=0, update_interval=1.0,
-                 max_shared_memory=1024 * 1024 * 1024, resume_file=None):
+                 max_shared_memory=1024 * 1024 * 1024, resume_file=None, dl_timeout=10):
         super().__init__(name='DLManager')
         self.log = logging.getLogger('DLM')
         self.proc_debug = False
@@ -40,6 +40,7 @@ class DLManager(Process):
         self.writer_result_q = None
         self.max_jobs = max_jobs
         self.max_workers = max_workers if max_workers else min(cpu_count() * 2, 16)
+        self.dl_timeout = dl_timeout
 
         # Analysis stuff
         self.analysis = None
@@ -574,7 +575,8 @@ class DLManager(Process):
         self.log.info(f'Starting download workers...')
         for i in range(self.max_workers):
             w = DLWorker(f'DLWorker {i + 1}', self.dl_worker_queue, self.dl_result_q,
-                         self.shared_memory.name, logging_queue=self.logging_queue)
+                         self.shared_memory.name, logging_queue=self.logging_queue,
+                         dl_timeout=self.dl_timeout)
             self.children.append(w)
             w.start()
 

legendary/downloader/workers.py

@@ -16,7 +16,8 @@ from legendary.models.downloading import DownloaderTaskResult, WriterTaskResult
 
 
 class DLWorker(Process):
-    def __init__(self, name, queue, out_queue, shm, max_retries=5, logging_queue=None):
+    def __init__(self, name, queue, out_queue, shm, max_retries=5,
+                 logging_queue=None, dl_timeout=10):
         super().__init__(name=name)
         self.q = queue
         self.o_q = out_queue
@@ -28,6 +29,7 @@ class DLWorker(Process):
         self.shm = SharedMemory(name=shm)
         self.log_level = logging.getLogger().level
         self.logging_queue = logging_queue
+        self.dl_timeout = dl_timeout
 
     def run(self):
         # we have to fix up the logger before we can start
@@ -66,7 +68,7 @@ class DLWorker(Process):
                     dl_start = time.time()
 
                     try:
-                        r = self.session.get(job.url, timeout=5.0)
+                        r = self.session.get(job.url, timeout=self.dl_timeout)
                         r.raise_for_status()
                     except Exception as e:
                         logger.warning(f'[{self.name}] Chunk download failed ({e!r}), retrying...')