fix: Download savegame chunks in parallel and fix savegame download when more than 1000 chunks in manifest

legendary/api/egs.py

@@ -2,11 +2,14 @@
 # coding: utf-8
 
 import urllib.parse
+from urllib3.util import Retry
 
 import requests
 import requests.adapters
 import logging
 
+from requests_futures.sessions import FuturesSession
+
 from requests.auth import HTTPBasicAuth
 
 from legendary.models.exceptions import InvalidCredentialsError
@@ -42,9 +45,22 @@ class EPCAPI:
         # increase maximum pool size for multithreaded metadata requests
         self.session.mount('https://', requests.adapters.HTTPAdapter(pool_maxsize=16))
 
+        retries = Retry(
+            total=3,
+            backoff_factor=0.1,
+            status_forcelist=[500, 501, 502, 503, 504],
+            allowed_methods={'GET'}
+        )
+
         self.unauth_session = requests.session()
         self.unauth_session.headers['User-Agent'] = self._user_agent
 
+        self.unauth_session.mount('https://', requests.adapters.HTTPAdapter(pool_connections=16, pool_maxsize=16, max_retries=retries))
+
+        self.unauth_future_session = FuturesSession(session=self.unauth_session,max_workers=16)
+        self.unauth_future_session.headers['User-Agent'] = self._user_agent
+
+
         self._oauth_basic = HTTPBasicAuth(self._user_basic, self._pw_basic)
 
         self.access_token = None
@@ -67,6 +83,7 @@ class EPCAPI:
             self._store_user_agent = f'EpicGamesLauncher/{version}'
             self.session.headers['User-Agent'] = self._user_agent
             self.unauth_session.headers['User-Agent'] = self._user_agent
+            self.unauth_future_session.headers['User-Agent'] = self._user_agent
         # update label
         if label := egs_params['label']:
             self._label = label

legendary/core.py

@@ -1034,11 +1034,9 @@ class LegendaryCore:
             os.makedirs(save_path)
 
         _save_dir = save_dir
-        savegames = self.egs.get_user_cloud_saves(app_name=app_name)
-        files = savegames['files']
+        manifests = self.egs.get_user_cloud_saves(app_name=app_name,manifests=True)
+        files = manifests["files"]
         for fname, f in files.items():
-            if '.manifest' not in fname:
-                continue
             f_parts = fname.split('/')
 
             if manifest_name and f_parts[4] != manifest_name:
@@ -1079,17 +1077,29 @@ class LegendaryCore:
             m = self.load_manifest(r.content)
 
             # download chunks required for extraction
-            chunks = dict()
+            chunkPaths = []
             for chunk in m.chunk_data_list.elements:
-                cpath_p = fname.split('/', 3)[:3]
-                cpath_p.append(chunk.path)
-                cpath = '/'.join(cpath_p)
-                if cpath not in files:
-                    self.log.warning(f'Chunk {cpath} not in file list, save data may be incomplete!')
-                    continue
+                chunkPaths.append(chunk.path)
 
-                self.log.debug(f'Downloading chunk "{cpath}"')
-                r = self.egs.unauth_session.get(files[cpath]['readLink'])
+            chunkSaves = self.egs.get_user_cloud_saves(app_name=app_name, filenames=chunkPaths, manifests=False)
+            chunkFiles = chunkSaves["files"]
+
+            if len(chunkFiles) != len(chunkPaths):
+                self.log.warning(f'Expected {len(chunkPaths)} chunks, but only found {len(chunkFiles)}! Save may be incomplete.')
+                continue
+
+            chunkLinks = [chunkFiles[c]['readLink'] for c in chunkFiles]
+            self.log.info(f'Downloading {len(chunkLinks)} chunks...')
+
+            def log_download(r, *args, **kwargs):
+                self.log.debug(f'Downloaded chunk {'/'.join(urlparse(r.url).path.split('/')[5:])} successfully.')
+
+            # map chunkLinks to self.egs.unauth_future_session.get
+            futures = [self.egs.unauth_future_session.get(link, hooks={'response': log_download}) for link in chunkLinks]
+
+            chunks = dict()
+            for future in futures:
+                r = future.result()
                 if r.status_code != 200:
                     self.log.error(f'Download failed, status code: {r.status_code}')
                     break

requirements.txt

@@ -1,2 +1,3 @@
 requests<3.0
+requests_futures
 filelock