From df55f18a789ff0fb7c4a296ed9f454548d443a18 Mon Sep 17 00:00:00 2001 From: derrod Date: Mon, 4 May 2020 13:59:57 +0200 Subject: [PATCH] [cli/core/downloader] Increase timeout and make it configurable Fixes #16 --- legendary/cli.py | 5 ++++- legendary/core.py | 6 ++++-- legendary/downloader/manager.py | 6 ++++-- legendary/downloader/workers.py | 6 ++++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index f756410..fd7e331 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -286,7 +286,8 @@ class LegendaryCLI: file_prefix_filter=args.file_prefix, file_exclude_filter=args.file_exclude_prefix, file_install_tag=args.install_tag, - dl_optimizations=args.order_opt) + dl_optimizations=args.order_opt, + dl_timeout=args.dl_timeout) # game is either up to date or hasn't changed, so we have nothing to do if not analysis.dl_size: @@ -471,6 +472,8 @@ def main(): type=str, help='Only download files with the specified install tag (testing)') install_parser.add_argument('--enable-reordering', dest='order_opt', action='store_true', help='Enable reordering to attempt to optimize RAM usage during download') + install_parser.add_argument('--dl-timeout', dest='dl_timeout', action='store', metavar='', type=int, + help='Connection timeout for downloader (default: 10 seconds)') launch_parser.add_argument('--offline', dest='offline', action='store_true', default=False, help='Skip login and launch game without online authentication') diff --git a/legendary/core.py b/legendary/core.py index f127f85..7e3df4d 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -363,7 +363,8 @@ class LegendaryCore: override_old_manifest: str = '', override_base_url: str = '', platform_override: str = '', file_prefix_filter: str = '', file_exclude_filter: str = '', file_install_tag: str = '', - dl_optimizations: bool = False) -> (DLManager, AnalysisResult, ManifestMeta): + dl_optimizations: bool = False, dl_timeout: int = 10 + ) -> (DLManager, AnalysisResult, ManifestMeta): # load old manifest old_manifest = None @@ -436,7 +437,8 @@ class LegendaryCore: process_opt = False dlm = DLManager(install_path, base_url, resume_file=resume_file, status_q=status_q, - max_shared_memory=max_shm * 1024 * 1024, max_workers=max_workers) + max_shared_memory=max_shm * 1024 * 1024, max_workers=max_workers, + dl_timeout=dl_timeout) anlres = dlm.run_analysis(manifest=new_manifest, old_manifest=old_manifest, patch=not disable_patching, resume=not force, file_prefix_filter=file_prefix_filter, diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 59069b5..a93dbc1 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -23,7 +23,7 @@ from legendary.models.manifest import ManifestComparison, Manifest class DLManager(Process): def __init__(self, download_dir, base_url, cache_dir=None, status_q=None, max_jobs=100, max_failures=5, max_workers=0, update_interval=1.0, - max_shared_memory=1024 * 1024 * 1024, resume_file=None): + max_shared_memory=1024 * 1024 * 1024, resume_file=None, dl_timeout=10): super().__init__(name='DLManager') self.log = logging.getLogger('DLM') self.proc_debug = False @@ -40,6 +40,7 @@ class DLManager(Process): self.writer_result_q = None self.max_jobs = max_jobs self.max_workers = max_workers if max_workers else min(cpu_count() * 2, 16) + self.dl_timeout = dl_timeout # Analysis stuff self.analysis = None @@ -574,7 +575,8 @@ class DLManager(Process): self.log.info(f'Starting download workers...') for i in range(self.max_workers): w = DLWorker(f'DLWorker {i + 1}', self.dl_worker_queue, self.dl_result_q, - self.shared_memory.name, logging_queue=self.logging_queue) + self.shared_memory.name, logging_queue=self.logging_queue, + dl_timeout=self.dl_timeout) self.children.append(w) w.start() diff --git a/legendary/downloader/workers.py b/legendary/downloader/workers.py index 3a8f137..0e2f7ae 100644 --- a/legendary/downloader/workers.py +++ b/legendary/downloader/workers.py @@ -16,7 +16,8 @@ from legendary.models.downloading import DownloaderTaskResult, WriterTaskResult class DLWorker(Process): - def __init__(self, name, queue, out_queue, shm, max_retries=5, logging_queue=None): + def __init__(self, name, queue, out_queue, shm, max_retries=5, + logging_queue=None, dl_timeout=10): super().__init__(name=name) self.q = queue self.o_q = out_queue @@ -28,6 +29,7 @@ class DLWorker(Process): self.shm = SharedMemory(name=shm) self.log_level = logging.getLogger().level self.logging_queue = logging_queue + self.dl_timeout = dl_timeout def run(self): # we have to fix up the logger before we can start @@ -66,7 +68,7 @@ class DLWorker(Process): dl_start = time.time() try: - r = self.session.get(job.url, timeout=5.0) + r = self.session.get(job.url, timeout=self.dl_timeout) r.raise_for_status() except Exception as e: logger.warning(f'[{self.name}] Chunk download failed ({e!r}), retrying...')