diff --git a/legendary/api/egs.py b/legendary/api/egs.py index 025fc77..46f6640 100644 --- a/legendary/api/egs.py +++ b/legendary/api/egs.py @@ -2,11 +2,14 @@ # coding: utf-8 import urllib.parse +from urllib3.util import Retry import requests import requests.adapters import logging +from requests_futures.sessions import FuturesSession + from requests.auth import HTTPBasicAuth from legendary.models.exceptions import InvalidCredentialsError @@ -42,9 +45,22 @@ class EPCAPI: # increase maximum pool size for multithreaded metadata requests self.session.mount('https://', requests.adapters.HTTPAdapter(pool_maxsize=16)) + retries = Retry( + total=3, + backoff_factor=0.1, + status_forcelist=[500, 501, 502, 503, 504], + allowed_methods={'GET'} + ) + self.unauth_session = requests.session() self.unauth_session.headers['User-Agent'] = self._user_agent + self.unauth_session.mount('https://', requests.adapters.HTTPAdapter(pool_connections=16, pool_maxsize=16, max_retries=retries)) + + self.unauth_future_session = FuturesSession(session=self.unauth_session,max_workers=16) + self.unauth_future_session.headers['User-Agent'] = self._user_agent + + self._oauth_basic = HTTPBasicAuth(self._user_basic, self._pw_basic) self.access_token = None @@ -67,6 +83,7 @@ class EPCAPI: self._store_user_agent = f'EpicGamesLauncher/{version}' self.session.headers['User-Agent'] = self._user_agent self.unauth_session.headers['User-Agent'] = self._user_agent + self.unauth_future_session.headers['User-Agent'] = self._user_agent # update label if label := egs_params['label']: self._label = label diff --git a/legendary/cli.py b/legendary/cli.py old mode 100644 new mode 100755 diff --git a/legendary/core.py b/legendary/core.py index 04344e3..2dbbadc 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -1034,11 +1034,9 @@ class LegendaryCore: os.makedirs(save_path) _save_dir = save_dir - savegames = self.egs.get_user_cloud_saves(app_name=app_name) - files = savegames['files'] + manifests = self.egs.get_user_cloud_saves(app_name=app_name,manifests=True) + files = manifests["files"] for fname, f in files.items(): - if '.manifest' not in fname: - continue f_parts = fname.split('/') if manifest_name and f_parts[4] != manifest_name: @@ -1079,17 +1077,29 @@ class LegendaryCore: m = self.load_manifest(r.content) # download chunks required for extraction - chunks = dict() + chunkPaths = [] for chunk in m.chunk_data_list.elements: - cpath_p = fname.split('/', 3)[:3] - cpath_p.append(chunk.path) - cpath = '/'.join(cpath_p) - if cpath not in files: - self.log.warning(f'Chunk {cpath} not in file list, save data may be incomplete!') - continue + chunkPaths.append(chunk.path) - self.log.debug(f'Downloading chunk "{cpath}"') - r = self.egs.unauth_session.get(files[cpath]['readLink']) + chunkSaves = self.egs.get_user_cloud_saves(app_name=app_name, filenames=chunkPaths, manifests=False) + chunkFiles = chunkSaves["files"] + + if len(chunkFiles) != len(chunkPaths): + self.log.warning(f'Expected {len(chunkPaths)} chunks, but only found {len(chunkFiles)}! Save may be incomplete.') + continue + + chunkLinks = [chunkFiles[c]['readLink'] for c in chunkFiles] + self.log.info(f'Downloading {len(chunkLinks)} chunks...') + + def log_download(r, *args, **kwargs): + self.log.debug(f'Downloaded chunk {'/'.join(urlparse(r.url).path.split('/')[5:])} successfully.') + + # map chunkLinks to self.egs.unauth_future_session.get + futures = [self.egs.unauth_future_session.get(link, hooks={'response': log_download}) for link in chunkLinks] + + chunks = dict() + for future in futures: + r = future.result() if r.status_code != 200: self.log.error(f'Download failed, status code: {r.status_code}') break diff --git a/requirements.txt b/requirements.txt index 481d045..2768244 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests<3.0 +requests_futures filelock