From 5b4a6e6d9f6652c597e7e95eef547f0bc180f558 Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 6 Aug 2020 09:28:25 +0200 Subject: [PATCH 001/101] [downloader] Fix temporary file name in result handler --- legendary/downloader/manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 6d41c6b..816307b 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -524,6 +524,9 @@ class DLManager(Process): self.num_tasks_processed_since_last += 1 if res.closed and self.resume_file and res.success: + if res.filename.endswith('.tmp'): + res.filename = res.filename[:-4] + file_hash = self.hash_map[res.filename] # write last completed file to super simple resume file with open(self.resume_file, 'ab') as rf: From f4a1e4610bd6ea05a468fab97364f540cf00bb8c Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 6 Aug 2020 09:28:35 +0200 Subject: [PATCH 002/101] [core] Skip savegame download if manifest empty --- legendary/core.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/legendary/core.py b/legendary/core.py index fc541ef..be362ea 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -513,6 +513,11 @@ class LegendaryCore: if r.status_code != 200: self.log.error(f'Download failed, status code: {r.status_code}') continue + + if not r.content: + self.log.error('Manifest is empty! Skipping...') + continue + m = self.load_manifest(r.content) # download chunks required for extraction From e1fc3df180a8761ba6539b22c74ee3b9e86cceb9 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 5 Sep 2020 04:51:20 +0200 Subject: [PATCH 003/101] [manager] Improve chunk reuse algorithm This will mostly be important for delta manifests, which are yet to be implemented. --- legendary/downloader/manager.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 816307b..772aa02 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -273,18 +273,21 @@ class DLManager(Process): old_file = old_manifest.file_manifest_list.get_file_by_path(changed) new_file = manifest.file_manifest_list.get_file_by_path(changed) - existing_chunks = dict() + existing_chunks = defaultdict(list) off = 0 for cp in old_file.chunk_parts: - existing_chunks[(cp.guid_num, cp.offset, cp.size)] = off + existing_chunks[cp.guid_num].append((off, cp.offset, cp.offset + cp.size)) off += cp.size for cp in new_file.chunk_parts: key = (cp.guid_num, cp.offset, cp.size) - if key in existing_chunks: - references[cp.guid_num] -= 1 - re_usable[changed][key] = existing_chunks[key] - analysis_res.reuse_size += cp.size + for file_o, cp_o, cp_end_o in existing_chunks[cp.guid_num]: + # check if new chunk part is wholly contained in the old chunk part + if cp_o <= cp.offset and (cp.offset + cp.size) <= cp_end_o: + references[cp.guid_num] -= 1 + re_usable[changed][key] = file_o + analysis_res.reuse_size += cp.size + break last_cache_size = current_cache_size = 0 # set to determine whether a file is currently cached or not From 4b4483c580f6498cc782399ad10ec65f3270afc7 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 5 Sep 2020 05:05:21 +0200 Subject: [PATCH 004/101] [core] Implement delta manifests --- legendary/core.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/legendary/core.py b/legendary/core.py index be362ea..ac30e42 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -641,6 +641,14 @@ class LegendaryCore: return new_manifest_data, base_urls + def get_delta_manifest(self, base_url, old_build_id, new_build_id): + """Get optimized delta manifest (doesn't seem to exist for most games)""" + r = self.egs.unauth_session.get(f'{base_url}/Deltas/{new_build_id}/{old_build_id}.delta') + if r.status_code == 200: + return r.content + else: + return None + def prepare_download(self, game: Game, base_game: Game = None, base_path: str = '', status_q: Queue = None, max_shm: int = 0, max_workers: int = 0, force: bool = False, disable_patching: bool = False, @@ -691,6 +699,17 @@ class LegendaryCore: # save manifest with version name as well for testing/downgrading/etc. self.lgd.save_manifest(game.app_name, new_manifest_data, version=new_manifest.meta.build_version) + # also fetch optimized delta manifest (may not exist) + if old_manifest and new_manifest and not (override_old_manifest or override_manifest): + delta_manifest_data = self.get_delta_manifest(randchoice(base_urls), + old_manifest.meta.build_id, + new_manifest.meta.build_id) + if delta_manifest_data: + delta_manifest = self.load_manifest(delta_manifest_data) + self.log.info(f'Using optimized delta manifest to upgrade from build' + f'"{old_manifest.meta.build_id}" to' + f'"{new_manifest.meta.build_id}"...') + new_manifest = delta_manifest # reuse existing installation's directory if igame := self.get_installed_game(base_game.app_name if base_game else game.app_name): From d1b8412b273b5e961a9a9184a76a33064ab3b4ce Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 5 Sep 2020 05:16:05 +0200 Subject: [PATCH 005/101] [core] Add options for delta manifests and update-when-repairing --- legendary/core.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index ac30e42..dab9931 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -657,7 +657,8 @@ class LegendaryCore: platform_override: str = '', file_prefix_filter: list = None, file_exclude_filter: list = None, file_install_tag: list = None, dl_optimizations: bool = False, dl_timeout: int = 10, - repair: bool = False, egl_guid: str = '' + repair: bool = False, repair_use_latest: bool = False, + disable_delta: bool = False, egl_guid: str = '' ) -> (DLManager, AnalysisResult, ManifestMeta): # load old manifest old_manifest = None @@ -700,7 +701,7 @@ class LegendaryCore: self.lgd.save_manifest(game.app_name, new_manifest_data, version=new_manifest.meta.build_version) # also fetch optimized delta manifest (may not exist) - if old_manifest and new_manifest and not (override_old_manifest or override_manifest): + if old_manifest and new_manifest and not (override_old_manifest or override_manifest or disable_delta): delta_manifest_data = self.get_delta_manifest(randchoice(base_urls), old_manifest.meta.build_id, new_manifest.meta.build_id) @@ -739,9 +740,11 @@ class LegendaryCore: self.log.info(f'Install path: {install_path}') if repair: - # use installed manifest for repairs, do not update to latest version (for now) - new_manifest = old_manifest - old_manifest = None + if not repair_use_latest: + # use installed manifest for repairs instead of updating + new_manifest = old_manifest + old_manifest = None + filename = clean_filename(f'{game.app_name}.repair') resume_file = os.path.join(self.lgd.get_tmp_path(), filename) force = False From fb9e8d4138ba52b610d5b3c0731b04166c3fb816 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 5 Sep 2020 05:18:15 +0200 Subject: [PATCH 006/101] [core] Fix disk space check for initial install and add override --- legendary/core.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index dab9931..8ad6c05 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -802,7 +802,10 @@ class LegendaryCore: return dlm, anlres, igame @staticmethod - def check_installation_conditions(analysis: AnalysisResult, install: InstalledGame) -> ConditionCheckResult: + def check_installation_conditions(analysis: AnalysisResult, + install: InstalledGame, + updating: bool = False, + ignore_space_req: bool = False) -> ConditionCheckResult: results = ConditionCheckResult(failures=set(), warnings=set()) # if on linux, check for eac in the files @@ -824,12 +827,20 @@ class LegendaryCore: results.warnings.add('This game is not marked for offline use (may still work).') # check if enough disk space is free (dl size is the approximate amount the installation will grow) - min_disk_space = analysis.uncompressed_dl_size + analysis.biggest_file_size + min_disk_space = analysis.uncompressed_dl_size + if updating: + min_disk_space += analysis.biggest_file_size + _, _, free = shutil.disk_usage(os.path.split(install.install_path)[0]) if free < min_disk_space: free_mib = free / 1024 / 1024 required_mib = min_disk_space / 1024 / 1024 - results.failures.add(f'Not enough available disk space! {free_mib:.02f} MiB < {required_mib:.02f} MiB') + if ignore_space_req: + results.warnings.add(f'Potentially not enough available disk space! ' + f'{free_mib:.02f} MiB < {required_mib:.02f} MiB') + else: + results.failures.add(f'Not enough available disk space!' + f'{free_mib:.02f} MiB < {required_mib:.02f} MiB') return results From 6711897750613ca3e3f06ec4c14f2d4e61395589 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 5 Sep 2020 05:20:27 +0200 Subject: [PATCH 007/101] [cli] Add flags for new installer options '--repair-and-update' for updating when repairing (duh) '--ignore-free-space' to make the free space error a warning instead '--disable-delta-manifests' to disable the use of delta manifests Also closes #70 --- legendary/cli.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index 9d1c38e..1a2d5e0 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -552,7 +552,9 @@ class LegendaryCLI: file_install_tag=args.install_tag, dl_optimizations=args.order_opt, dl_timeout=args.dl_timeout, - repair=args.repair_mode) + repair=args.repair_mode, + repair_use_latest=args.repair_and_update, + disable_delta=args.disable_delta) # game is either up to date or hasn't changed, so we have nothing to do if not analysis.dl_size: @@ -574,7 +576,9 @@ class LegendaryCLI: logger.info(f'Reusable size: {analysis.reuse_size / 1024 / 1024:.02f} MiB (chunks) / ' f'{analysis.unchanged / 1024 / 1024:.02f} MiB (unchanged)') - res = self.core.check_installation_conditions(analysis=analysis, install=igame) + res = self.core.check_installation_conditions(analysis=analysis, install=igame, + updating=self.core.is_installed(args.app_name), + ignore_space_req=args.ignore_space) if res.failures: logger.fatal('Download cannot proceed, the following errors occured:') @@ -1031,6 +1035,12 @@ def main(): help='Set save game path to be used for sync-saves') install_parser.add_argument('--repair', dest='repair_mode', action='store_true', help='Repair installed game by checking and redownloading corrupted/missing files') + install_parser.add_argument('--repair-and-update', dest='repair_and_update', action='store_true', + help='Update game to the latest version when repairing') + install_parser.add_argument('--ignore-free-space', dest='ignore_space', action='store_true', + help='Do not abort if not enough free space is available') + install_parser.add_argument('--disable-delta-manifests', dest='disable_delta', action='store_true', + help='Do not use delta manfiests when updating (may increase download size)') launch_parser.add_argument('--offline', dest='offline', action='store_true', default=False, help='Skip login and launch game without online authentication') From ade9080152519185277cc51d52951c519687fc19 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 5 Sep 2020 05:21:36 +0200 Subject: [PATCH 008/101] [cli] Convert import path to absolute Fixes #61 --- legendary/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/legendary/cli.py b/legendary/cli.py index 1a2d5e0..bbee5d2 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -772,6 +772,9 @@ class LegendaryCLI: logger.info(f'Run "legendary repair {args.app_name}" to repair your game installation.') def import_game(self, args): + # make sure path is absolute + args.app_path = os.path.abspath(args.app_path) + if not os.path.exists(args.app_path): logger.error(f'Specified path "{args.app_path}" does not exist!') exit(1) From e9e40a37827ffe3faed52231018e46e582b225c7 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 5 Sep 2020 05:25:13 +0200 Subject: [PATCH 009/101] [cli] Add install size and path to CSV/TSV output Fixes #91 and closes #68 --- legendary/cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index bbee5d2..c294bac 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -183,9 +183,11 @@ class LegendaryCLI: if args.csv or args.tsv: writer = csv.writer(stdout, dialect='excel-tab' if args.tsv else 'excel') - writer.writerow(['App name', 'App title', 'Installed version', 'Available version', 'Update available']) + writer.writerow(['App name', 'App title', 'Installed version', 'Available version', + 'Update available', 'Install size', 'Install path']) writer.writerows((game.app_name, game.title, game.version, versions[game.app_name], - versions[game.app_name] != game.version) for game in games) + versions[game.app_name] != game.version, game.install_size, game.install_path) + for game in games) return print('\nInstalled games:') @@ -774,7 +776,7 @@ class LegendaryCLI: def import_game(self, args): # make sure path is absolute args.app_path = os.path.abspath(args.app_path) - + if not os.path.exists(args.app_path): logger.error(f'Specified path "{args.app_path}" does not exist!') exit(1) From 0430204f5958aac37f7ab5e9fdb128410bf3b272 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 5 Sep 2020 05:27:44 +0200 Subject: [PATCH 010/101] [core] Add debug message for delta manifest unavailability --- legendary/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/legendary/core.py b/legendary/core.py index 8ad6c05..56b7d2a 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -711,6 +711,8 @@ class LegendaryCore: f'"{old_manifest.meta.build_id}" to' f'"{new_manifest.meta.build_id}"...') new_manifest = delta_manifest + else: + self.log.debug(f'No Delta manifest received from CDN.') # reuse existing installation's directory if igame := self.get_installed_game(base_game.app_name if base_game else game.app_name): From bd66d6c5dcc4e44ffd4d1d7535c61a216e15afbe Mon Sep 17 00:00:00 2001 From: derrod Date: Sun, 6 Sep 2020 03:16:06 +0200 Subject: [PATCH 011/101] [api/core/downloader] Update User-Agents --- legendary/api/egs.py | 2 +- legendary/core.py | 4 ++-- legendary/downloader/workers.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/legendary/api/egs.py b/legendary/api/egs.py index 992b73c..e8549c2 100644 --- a/legendary/api/egs.py +++ b/legendary/api/egs.py @@ -10,7 +10,7 @@ from legendary.models.exceptions import InvalidCredentialsError class EPCAPI: - _user_agent = 'UELauncher/10.16.1-13343695+++Portal+Release-Live Windows/10.0.18363.1.256.64bit' + _user_agent = 'UELauncher/10.18.6-14188424+++Portal+Release-Live Windows/10.0.18363.1.256.64bit' # required for the oauth request _user_basic = '34a02cf8f4414e29b15921876da36f9a' _pw_basic = 'daafbccc737745039dffe53d94fc76cf' diff --git a/legendary/core.py b/legendary/core.py index 56b7d2a..a75d3e0 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -103,8 +103,8 @@ class LegendaryCore: 'X-Requested-With': 'XMLHttpRequest', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' - 'EpicGamesLauncher/10.16.1-13343695+++Portal+Release-Live ' - 'UnrealEngine/4.23.0-13343695+++Portal+Release-Live ' + 'EpicGamesLauncher/10.18.6-14188424+++Portal+Release-Live ' + 'UnrealEngine/4.23.0-14188424+++Portal+Release-Live ' 'Chrome/59.0.3071.15 Safari/537.36' }) diff --git a/legendary/downloader/workers.py b/legendary/downloader/workers.py index 6182ecf..3d9c558 100644 --- a/legendary/downloader/workers.py +++ b/legendary/downloader/workers.py @@ -22,7 +22,7 @@ class DLWorker(Process): self.o_q = out_queue self.session = requests.session() self.session.headers.update({ - 'User-Agent': 'EpicGamesLauncher/10.16.1-13343695+++Portal+Release-Live Windows/10.0.18363.1.256.64bit' + 'User-Agent': 'EpicGamesLauncher/10.18.6-14188424+++Portal+Release-Live Windows/10.0.18363.1.256.64bit' }) self.max_retries = max_retries self.shm = SharedMemory(name=shm) From a55f75d5e87044c4d4e7ea4640957bb6bd033909 Mon Sep 17 00:00:00 2001 From: derrod Date: Sun, 6 Sep 2020 05:57:28 +0200 Subject: [PATCH 012/101] [core/downloader] Prevent file deletion when using delta manifest This is technically not how we should do this. In theory we should "overlay" the delta manifest over the proper one and simply add/replace the chunk/file list entries with the one from the delta manifest. However simply not deleting files also works for the time being since files are rarely deleted anyways. --- legendary/core.py | 5 ++++- legendary/downloader/manager.py | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index a75d3e0..3662dd5 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -662,6 +662,7 @@ class LegendaryCore: ) -> (DLManager, AnalysisResult, ManifestMeta): # load old manifest old_manifest = None + delta_manifest_used = False # load old manifest if we have one if override_old_manifest: @@ -711,6 +712,7 @@ class LegendaryCore: f'"{old_manifest.meta.build_id}" to' f'"{new_manifest.meta.build_id}"...') new_manifest = delta_manifest + delta_manifest_used = True else: self.log.debug(f'No Delta manifest received from CDN.') @@ -782,7 +784,8 @@ class LegendaryCore: file_prefix_filter=file_prefix_filter, file_exclude_filter=file_exclude_filter, file_install_tag=file_install_tag, - processing_optimization=process_opt) + processing_optimization=process_opt, + delta_manifest_used=delta_manifest_used) prereq = None if new_manifest.meta.prereq_ids: diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 772aa02..60293a0 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -79,7 +79,7 @@ class DLManager(Process): def run_analysis(self, manifest: Manifest, old_manifest: Manifest = None, patch=True, resume=True, file_prefix_filter=None, file_exclude_filter=None, file_install_tag=None, - processing_optimization=False) -> AnalysisResult: + processing_optimization=False, delta_manifest_used=False) -> AnalysisResult: """ Run analysis on manifest and old manifest (if not None) and return a result with a summary resources required in order to install the provided manifest. @@ -92,6 +92,7 @@ class DLManager(Process): :param file_exclude_filter: Exclude files with this prefix from download :param file_install_tag: Only install files with the specified tag :param processing_optimization: Attempt to optimize processing order and RAM usage + :param delta_manifest_used: whether or not the manifest is a delta manifest :return: AnalysisResult """ @@ -181,6 +182,10 @@ class DLManager(Process): analysis_res.install_size = sum(fm.file_size for fm in manifest.file_manifest_list.elements if fm.filename in mc.added) + # todo properly handle delta manifests + if delta_manifest_used: + mc.removed = set() + if mc.removed: analysis_res.removed = len(mc.removed) self.log.debug(f'{analysis_res.removed} removed files') From 9c9fee8f11de8269daa5ce9c3473facabf7dddb8 Mon Sep 17 00:00:00 2001 From: derrod Date: Sun, 6 Sep 2020 06:18:43 +0200 Subject: [PATCH 013/101] [core] Don't request delta manifest if old == new --- legendary/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/legendary/core.py b/legendary/core.py index 3662dd5..b5bd487 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -702,7 +702,8 @@ class LegendaryCore: self.lgd.save_manifest(game.app_name, new_manifest_data, version=new_manifest.meta.build_version) # also fetch optimized delta manifest (may not exist) - if old_manifest and new_manifest and not (override_old_manifest or override_manifest or disable_delta): + if old_manifest and new_manifest and not (override_old_manifest or override_manifest or disable_delta or + old_manifest.meta.build_id == new_manifest.meta.build_id): delta_manifest_data = self.get_delta_manifest(randchoice(base_urls), old_manifest.meta.build_id, new_manifest.meta.build_id) From 7575b8f3a74b83f3dd8a24f2f27383ed8ce0a5b9 Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 8 Sep 2020 04:25:34 +0200 Subject: [PATCH 014/101] [cli/core] Prevent crash when game asset metadata is missing --- legendary/cli.py | 18 ++++++++++++++---- legendary/core.py | 5 ++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index c294bac..783e06d 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -179,7 +179,12 @@ class LegendaryCLI: versions = dict() for game in games: - versions[game.app_name] = self.core.get_asset(game.app_name).build_version + try: + versions[game.app_name] = self.core.get_asset(game.app_name).build_version + except ValueError: + logger.warning(f'Metadata for "{game.app_name}" is missing, the game may have been removed from ' + f'your account or not be in legendary\'s database yet, try rerunning the command ' + f'with "--check-updates".') if args.csv or args.tsv: writer = csv.writer(stdout, dialect='excel-tab' if args.tsv else 'excel') @@ -187,7 +192,7 @@ class LegendaryCLI: 'Update available', 'Install size', 'Install path']) writer.writerows((game.app_name, game.title, game.version, versions[game.app_name], versions[game.app_name] != game.version, game.install_size, game.install_path) - for game in games) + for game in games if game.app_name in versions) return print('\nInstalled games:') @@ -204,7 +209,7 @@ class LegendaryCLI: print(f' + Location: {game.install_path}') if not os.path.exists(game.install_path): print(f' ! Game does no longer appear to be installed (directory "{game.install_path}" missing)!') - elif versions[game.app_name] != game.version: + elif game.app_name in versions and versions[game.app_name] != game.version: print(f' -> Update available! Installed: {game.version}, Latest: {versions[game.app_name]}') print(f'\nTotal: {len(games)}') @@ -425,7 +430,12 @@ class LegendaryCLI: if not args.skip_version_check and not self.core.is_noupdate_game(app_name): logger.info('Checking for updates...') - latest = self.core.get_asset(app_name, update=True) + try: + latest = self.core.get_asset(app_name, update=True) + except ValueError: + logger.fatal(f'Metadata for "{app_name}" does not exist, cannot launch!') + exit(1) + if latest.build_version != igame.version: logger.error('Game is out of date, please update or launch with update check skipping!') exit(1) diff --git a/legendary/core.py b/legendary/core.py index b5bd487..19bd6f4 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -204,7 +204,10 @@ class LegendaryCore: if update: self.get_assets(update_assets=True) - return next(i for i in self.lgd.assets if i.app_name == app_name) + try: + return next(i for i in self.lgd.assets if i.app_name == app_name) + except StopIteration: + raise ValueError def get_game(self, app_name, update_meta=False) -> Game: if update_meta: From 6bae5d3081180aae6891209a194fd1d2bda6aaeb Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 8 Sep 2020 04:38:11 +0200 Subject: [PATCH 015/101] [core] Prevent importing unknown games from EGL --- legendary/core.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/legendary/core.py b/legendary/core.py index 19bd6f4..57f6b80 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -209,6 +209,9 @@ class LegendaryCore: except StopIteration: raise ValueError + def asset_valid(self, app_name) -> bool: + return any(i.app_name == app_name for i in self.lgd.assets) + def get_game(self, app_name, update_meta=False) -> Game: if update_meta: self.get_game_list(True) @@ -965,7 +968,9 @@ class LegendaryCore: def egl_get_importable(self): return [g for g in self.egl.get_manifests() - if not self.is_installed(g.app_name) and g.main_game_appname == g.app_name] + if not self.is_installed(g.app_name) and + g.main_game_appname == g.app_name and + self.asset_valid(g.app_name)] def egl_get_exportable(self): if not self.egl.manifests: @@ -973,6 +978,9 @@ class LegendaryCore: return [g for g in self.get_installed_list() if g.app_name not in self.egl.manifests] def egl_import(self, app_name): + if not self.asset_valid(app_name): + raise ValueError(f'To-be-imported game {app_name} not in game asset database!') + self.log.debug(f'Importing "{app_name}" from EGL') # load egl json file try: @@ -1094,6 +1102,8 @@ class LegendaryCore: for egl_igame in self.egl.get_manifests(): if egl_igame.main_game_appname != egl_igame.app_name: # skip DLC continue + if not self.asset_valid(egl_igame.app_name): # skip non-owned games + continue if not self._is_installed(egl_igame.app_name): self.egl_import(egl_igame.app_name) From 5b855b0d3effa859a47ad868ed5626b19679c84f Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 8 Sep 2020 04:38:29 +0200 Subject: [PATCH 016/101] [cli] Add note about potentially missing games when importing --- legendary/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/legendary/cli.py b/legendary/cli.py index 783e06d..f21c2bd 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -922,6 +922,9 @@ class LegendaryCLI: for egl_game in importable: print(' *', egl_game.app_name, '-', egl_game.display_name) + print('\nNote: Only games that are also in Legendary\'s database are listed, ' + 'if anything is missing run "list-games" first to update it.') + if args.yes or get_boolean_choice('Do you want to import the games from EGL?'): for egl_game in importable: logger.info(f'Importing "{egl_game.display_name}"...') From acb7476a229bb304b897e6a496d72756d89bc1e0 Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 8 Sep 2020 04:50:14 +0200 Subject: [PATCH 017/101] [cli] Add basic "status" command --- legendary/cli.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/legendary/cli.py b/legendary/cli.py index f21c2bd..55d90d3 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -957,6 +957,18 @@ class LegendaryCLI: else: self.core.egl_sync() + def status(self, args): + if not args.offline: + if not self.core.login(): + logger.error('Log in failed!') + exit(1) + + print(f'Epic account: {self.core.lgd.userdata["displayName"]}') + print(f'Games available: {len(self.core.get_game_list(update_assets=not args.offline))}') + print(f'Games installed: {len(self.core.get_installed_list())}') + print(f'EGL Sync enabled: {self.core.egl_sync_enabled}') + print(f'Config directory: {self.core.lgd.path}') + def main(): parser = argparse.ArgumentParser(description=f'Legendary v{__version__} - "{__codename__}"') @@ -986,6 +998,7 @@ def main(): verify_parser = subparsers.add_parser('verify-game', help='Verify a game\'s local files') import_parser = subparsers.add_parser('import-game', help='Import an already installed game') egl_sync_parser = subparsers.add_parser('egl-sync', help='Setup or run Epic Games Launcher sync') + status_parser = subparsers.add_parser('status', help='Show legendary status information') install_parser.add_argument('app_name', help='Name of the app', metavar='') uninstall_parser.add_argument('app_name', help='Name of the app', metavar='') @@ -1159,6 +1172,9 @@ def main(): egl_sync_parser.add_argument('--unlink', dest='unlink', action='store_true', help='Disable sync and remove EGL metadata from installed games') + status_parser.add_argument('--offline', dest='offline', action='store_true', + help='Only print offline status information, do not login') + args, extra = parser.parse_known_args() if args.version: @@ -1168,7 +1184,7 @@ def main(): if args.subparser_name not in ('auth', 'list-games', 'list-installed', 'list-files', 'launch', 'download', 'uninstall', 'install', 'update', 'repair', 'list-saves', 'download-saves', 'sync-saves', - 'verify-game', 'import-game', 'egl-sync'): + 'verify-game', 'import-game', 'egl-sync', 'status'): print(parser.format_help()) # Print the main help *and* the help for all of the subcommands. Thanks stackoverflow! @@ -1225,6 +1241,8 @@ def main(): cli.import_game(args) elif args.subparser_name == 'egl-sync': cli.egs_sync(args) + elif args.subparser_name == 'status': + cli.status(args) except KeyboardInterrupt: logger.info('Command was aborted via KeyboardInterrupt, cleaning up...') From d4f4571f859144711cb631474142f0cfc7ae915b Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 8 Sep 2020 04:56:56 +0200 Subject: [PATCH 018/101] [core] Allow "wrapper" and "no_wine" in "default" section The entire config crap will have to be rewritten to be "nicer"... --- legendary/core.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index 57f6b80..e681c0b 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -316,7 +316,9 @@ class LegendaryCore: params = [] - if wrapper or (wrapper := self.lgd.config.get(app_name, 'wrapper', fallback=None)): + if wrapper or (wrapper := self.lgd.config.get(app_name, 'wrapper', + fallback=self.lgd.config.get('default', 'wrapper', + fallback=None))): params.extend(shlex.split(wrapper)) if os.name != 'nt' and not disable_wine: @@ -326,7 +328,8 @@ class LegendaryCore: # check if there's a game specific override wine_bin = self.lgd.config.get(app_name, 'wine_executable', fallback=wine_bin) - if not self.lgd.config.getboolean(app_name, 'no_wine', fallback=False): + if not self.lgd.config.getboolean(app_name, 'no_wine', + fallback=self.lgd.config.get('default', 'no_wine', fallback=False)): params.append(wine_bin) params.append(game_exe) From 6f53964b491ad4d93c9dbcb54d0503738f6c1a9e Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 8 Sep 2020 05:12:14 +0200 Subject: [PATCH 019/101] [cli] Add --json output format for some commands --- legendary/cli.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/legendary/cli.py b/legendary/cli.py index 55d90d3..3b5fb8f 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -3,6 +3,7 @@ import argparse import csv +import json import logging import os import shlex @@ -158,6 +159,16 @@ class LegendaryCLI: writer.writerow((dlc.app_name, dlc.app_title, dlc.app_version, True)) return + if args.json: + _out = [] + for game in games: + _j = vars(game) + _j['dlcs'] = [vars(dlc) for dlc in dlc_list[game.asset_info.catalog_item_id]] + _out.append(_j) + + print(json.dumps(_out, sort_keys=True, indent=2)) + return + print('\nAvailable games:') for game in games: print(f' * {game.app_title} (App name: {game.app_name} | Version: {game.app_version})') @@ -195,6 +206,10 @@ class LegendaryCLI: for game in games if game.app_name in versions) return + if args.json: + print(json.dumps([vars(g) for g in games], indent=2, sort_keys=True)) + return + print('\nInstalled games:') for game in games: if game.install_size == 0: @@ -251,6 +266,17 @@ class LegendaryCLI: writer = csv.writer(stdout, dialect='excel-tab' if args.tsv else 'excel') writer.writerow(['path', 'hash', 'size', 'install_tags']) writer.writerows((fm.filename, fm.hash.hex(), fm.file_size, '|'.join(fm.install_tags))for fm in files) + elif args.json: + _files = [] + for fm in files: + _files.append(dict( + filename=fm.filename, + sha_hash=fm.hash.hex(), + install_tags=fm.install_tags, + file_size=fm.file_size, + flags=fm.flags, + )) + print(json.dumps(_files, sort_keys=True, indent=2)) else: install_tags = set() for fm in files: @@ -963,6 +989,16 @@ class LegendaryCLI: logger.error('Log in failed!') exit(1) + if args.json: + print(json.dumps(dict( + account=self.core.lgd.userdata["displayName"], + games_available=len(self.core.get_game_list(update_assets=not args.offline)), + games_installed=len(self.core.get_installed_list()), + egl_sync_enabled=self.core.egl_sync_enabled, + config_directory=self.core.lgd.path + ), indent=2, sort_keys=True)) + return + print(f'Epic account: {self.core.lgd.userdata["displayName"]}') print(f'Games available: {len(self.core.get_game_list(update_assets=not args.offline))}') print(f'Games installed: {len(self.core.get_installed_list())}') @@ -1114,6 +1150,7 @@ def main(): help='Also include Unreal Engine content (Engine/Marketplace) in list') list_parser.add_argument('--csv', dest='csv', action='store_true', help='List games in CSV format') list_parser.add_argument('--tsv', dest='tsv', action='store_true', help='List games in TSV format') + list_parser.add_argument('--json', dest='json', action='store_true', help='List games in JSON format') list_installed_parser.add_argument('--check-updates', dest='check_updates', action='store_true', help='Check for updates for installed games') @@ -1121,6 +1158,8 @@ def main(): help='List games in CSV format') list_installed_parser.add_argument('--tsv', dest='tsv', action='store_true', help='List games in TSV format') + list_installed_parser.add_argument('--json', dest='json', action='store_true', + help='List games in JSON format') list_installed_parser.add_argument('--show-dirs', dest='include_dir', action='store_true', help='Print installation directory in output') @@ -1132,6 +1171,7 @@ def main(): help='Manifest URL or path to use instead of the CDN one') list_files_parser.add_argument('--csv', dest='csv', action='store_true', help='Output in CSV format') list_files_parser.add_argument('--tsv', dest='tsv', action='store_true', help='Output in TSV format') + list_files_parser.add_argument('--json', dest='json', action='store_true', help='Output in JSON format') list_files_parser.add_argument('--hashlist', dest='hashlist', action='store_true', help='Output file hash list in hashcheck/sha1sum -c compatible format') list_files_parser.add_argument('--install-tag', dest='install_tag', action='store', metavar='', @@ -1174,6 +1214,8 @@ def main(): status_parser.add_argument('--offline', dest='offline', action='store_true', help='Only print offline status information, do not login') + status_parser.add_argument('--json', dest='json', action='store_true', + help='Show status in JSON format') args, extra = parser.parse_known_args() From 53a818bedc2d40f593aef5b947d7fea6bdf7759c Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 8 Sep 2020 05:14:51 +0200 Subject: [PATCH 020/101] Update README.md --- README.md | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4a762d9..bb30303 100644 --- a/README.md +++ b/README.md @@ -135,9 +135,9 @@ $ legendary -y egl-sync ## Usage ```` -usage: legendary [-h] [-v] [-y] [-V] {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync} ... +usage: legendary [-h] [-v] [-y] [-V] {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync,status} ... -Legendary v0.0.X - "Codename" +Legendary v0.X.X - "Codename" optional arguments: -h, --help show this help message and exit @@ -146,7 +146,7 @@ optional arguments: -V Print version and exit Commands: - {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync} + {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync,status} auth Authenticate with EPIC install (download,update,repair) Download a game @@ -161,6 +161,7 @@ Commands: verify-game Verify a game's local files import-game Import an already installed game egl-sync Setup or run Epic Games Launcher sync + status Show legendary status information Individual command help: @@ -209,6 +210,10 @@ optional arguments: --dl-timeout Connection timeout for downloader (default: 10 seconds) --save-path Set save game path to be used for sync-saves --repair Repair installed game by checking and redownloading corrupted/missing files + --repair-and-update Update game to the latest version when repairing + --ignore-free-space Do not abort if not enough free space is available + --disable-delta-manifests + Do not use delta manfiests when updating (may increase download size) Command: uninstall @@ -249,7 +254,7 @@ optional arguments: Command: list-games -usage: legendary list-games [-h] [--platform ] [--include-ue] [--csv] [--tsv] +usage: legendary list-games [-h] [--platform ] [--include-ue] [--csv] [--tsv] [--json] optional arguments: -h, --help show this help message and exit @@ -258,21 +263,23 @@ optional arguments: --include-ue Also include Unreal Engine content (Engine/Marketplace) in list --csv List games in CSV format --tsv List games in TSV format + --json List games in JSON format Command: list-installed -usage: legendary list-installed [-h] [--check-updates] [--csv] [--tsv] [--show-dirs] +usage: legendary list-installed [-h] [--check-updates] [--csv] [--tsv] [--json] [--show-dirs] optional arguments: -h, --help show this help message and exit --check-updates Check for updates for installed games --csv List games in CSV format --tsv List games in TSV format + --json List games in JSON format --show-dirs Print installation directory in output Command: list-files -usage: legendary list-files [-h] [--force-download] [--platform ] [--manifest ] [--csv] [--tsv] [--hashlist] [--install-tag ] [] +usage: legendary list-files [-h] [--force-download] [--platform ] [--manifest ] [--csv] [--tsv] [--json] [--hashlist] [--install-tag ] [] positional arguments: Name of the app (optional) @@ -285,6 +292,7 @@ optional arguments: --manifest Manifest URL or path to use instead of the CDN one --csv Output in CSV format --tsv Output in TSV format + --json Output in JSON format --hashlist Output file hash list in hashcheck/sha1sum -c compatible format --install-tag Show only files with specified install tag @@ -363,6 +371,15 @@ optional arguments: --import-only Only import games from EGL (no export) --export-only Only export games to EGL (no import) --unlink Disable sync and remove EGL metadata from installed games + + +Command: status +usage: legendary status [-h] [--offline] [--json] + +optional arguments: + -h, --help show this help message and exit + --offline Only print offline status information, do not login + --json Show status in JSON format ```` @@ -427,7 +444,8 @@ DXVK_CONFIG_FILE = /mnt/tank/games/Game/dxvk.conf [AppName2] ; Use a wrapper to run this script -wrapper = /path/to/wrapper --parameters +; Note that the path might have to be quoted if it contains spaces +wrapper = "/path/to/Proton 5.0/proton" run ; Do not run this executable with WINE (e.g. when the wrapper handles that) no_wine = true ```` From 09b918d156444ecc432be9420b53552f87b0da26 Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 8 Sep 2020 05:15:23 +0200 Subject: [PATCH 021/101] Bump version to 0.20.0 --- legendary/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/legendary/__init__.py b/legendary/__init__.py index aeaa166..4dc2ee1 100644 --- a/legendary/__init__.py +++ b/legendary/__init__.py @@ -1,4 +1,4 @@ """Legendary!""" -__version__ = '0.0.19' -__codename__ = 'Interloper' +__version__ = '0.20.0' +__codename__ = 'Endgame' From 6b8838497b07d93933de0af0998dfbec376827c0 Mon Sep 17 00:00:00 2001 From: Alberto Oporto Ames Date: Tue, 8 Sep 2020 10:21:07 -0500 Subject: [PATCH 022/101] [CI] Add deb package (#59) * Add deb package to CI, #57 * Fix dependencies --- .github/workflows/python.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a7f94b7..93b82a9 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -47,3 +47,38 @@ jobs: with: name: ${{ runner.os }}-package path: legendary/dist/* + + deb: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ['ubuntu-20.04'] + + fail-fast: true + max-parallel: 3 + + steps: + - uses: actions/checkout@v2 + + - name: Dependencies + run: sudo apt install + python3-all + python3-stdeb + dh-python + python3-requests + python3-setuptools + python3-wheel + + - name: Build + run: python3 setup.py --command-packages=stdeb.command bdist_deb + + - name: Os version + id: os_version + run: | + source /etc/os-release + echo ::set-output name=version::$NAME-$VERSION_ID + + - uses: actions/upload-artifact@v2 + with: + name: ${{ steps.os_version.outputs.version }}-deb-package + path: deb_dist/*.deb From b7fd2031f98856b303a503cc8e6c689438646311 Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 8 Sep 2020 17:50:36 +0200 Subject: [PATCH 023/101] [cli] Fix status command when not logged in yet --- legendary/cli.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index 3b5fb8f..ade56b3 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -985,23 +985,34 @@ class LegendaryCLI: def status(self, args): if not args.offline: - if not self.core.login(): - logger.error('Log in failed!') - exit(1) + try: + if not self.core.login(): + logger.error('Log in failed!') + exit(1) + except ValueError: + pass + if not self.core.lgd.userdata: + user_name = '' + args.offline = True + else: + user_name = self.core.lgd.userdata['displayName'] + + games_available = len(self.core.get_game_list(update_assets=not args.offline)) + games_installed = len(self.core.get_installed_list()) if args.json: print(json.dumps(dict( - account=self.core.lgd.userdata["displayName"], - games_available=len(self.core.get_game_list(update_assets=not args.offline)), - games_installed=len(self.core.get_installed_list()), + account=user_name, + games_available=games_available, + games_installed=games_installed, egl_sync_enabled=self.core.egl_sync_enabled, config_directory=self.core.lgd.path ), indent=2, sort_keys=True)) return - print(f'Epic account: {self.core.lgd.userdata["displayName"]}') - print(f'Games available: {len(self.core.get_game_list(update_assets=not args.offline))}') - print(f'Games installed: {len(self.core.get_installed_list())}') + print(f'Epic account: {user_name}') + print(f'Games available: {games_available}') + print(f'Games installed: {games_installed}') print(f'EGL Sync enabled: {self.core.egl_sync_enabled}') print(f'Config directory: {self.core.lgd.path}') From b8b8a5d95384c5c09f97167a223e6342fed776f3 Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 8 Sep 2020 18:10:05 +0200 Subject: [PATCH 024/101] [cli] Add --version and --debug aliases for -V/-v --- legendary/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index ade56b3..e21e997 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -1022,9 +1022,9 @@ def main(): parser.register('action', 'parsers', AliasedSubParsersAction) # general arguments - parser.add_argument('-v', dest='debug', action='store_true', help='Set loglevel to debug') + parser.add_argument('-v', '--debug', dest='debug', action='store_true', help='Set loglevel to debug') parser.add_argument('-y', '--yes', dest='yes', action='store_true', help='Default to yes for all prompts') - parser.add_argument('-V', dest='version', action='store_true', help='Print version and exit') + parser.add_argument('-V', '--version', dest='version', action='store_true', help='Print version and exit') # all the commands subparsers = parser.add_subparsers(title='Commands', dest='subparser_name') From 515705c061b1b27afd918488894f1a751077b41f Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 9 Sep 2020 09:53:05 +0200 Subject: [PATCH 025/101] [cli] list-files: Error out of invalid AppName specified Closes #94 --- legendary/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/legendary/cli.py b/legendary/cli.py index e21e997..8b426cd 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -250,6 +250,9 @@ class LegendaryCLI: logger.error('Login failed! Cannot continue with download process.') exit(1) game = self.core.get_game(args.app_name, update_meta=True) + if not game: + logger.fatal(f'Could not fetch metadata for "{args.app_name}" (check spelling/account ownership)') + exit(1) manifest_data, _ = self.core.get_cdn_manifest(game, platform_override=args.platform_override) manifest = self.core.load_manifest(manifest_data) From 70ed243b32ca8c52aa56289ad98ad8072cccf154 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 9 Sep 2020 10:13:03 +0200 Subject: [PATCH 026/101] [utils] Add manifest combiner for delta manifests --- legendary/utils/manifests.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 legendary/utils/manifests.py diff --git a/legendary/utils/manifests.py b/legendary/utils/manifests.py new file mode 100644 index 0000000..aa85d44 --- /dev/null +++ b/legendary/utils/manifests.py @@ -0,0 +1,28 @@ +from legendary.models.manifest import Manifest + + +def combine_manifests(base_manifest: Manifest, delta_manifest: Manifest): + added = set() + # overwrite file elements with the ones from the delta manifest + for idx, file_elem in enumerate(base_manifest.file_manifest_list.elements): + try: + delta_file = delta_manifest.file_manifest_list.get_file_by_path(file_elem.filename) + base_manifest.file_manifest_list.elements[idx] = delta_file + added.add(delta_file.filename) + except ValueError: + pass + + # add other files that may be missing + for delta_file in delta_manifest.file_manifest_list.elements: + if delta_file.filename not in added: + base_manifest.file_manifest_list.elements.append(delta_file) + # update count and clear map + base_manifest.file_manifest_list.count = len(base_manifest.file_manifest_list.elements) + base_manifest.file_manifest_list._path_map = None + + # add chunks from delta manifest to main manifest and again clear path caches + base_manifest.chunk_data_list.elements.extend(delta_manifest.chunk_data_list.elements) + base_manifest.chunk_data_list.count = len(base_manifest.chunk_data_list.elements) + base_manifest.chunk_data_list._guid_map = None + base_manifest.chunk_data_list._guid_int_map = None + base_manifest.chunk_data_list._path_map = None From e898fe03fc73ba0a2abdf7c58af5107693aa1b85 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 9 Sep 2020 10:13:21 +0200 Subject: [PATCH 027/101] [downloader] Remove delta manifest workarounds --- legendary/downloader/manager.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 60293a0..772aa02 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -79,7 +79,7 @@ class DLManager(Process): def run_analysis(self, manifest: Manifest, old_manifest: Manifest = None, patch=True, resume=True, file_prefix_filter=None, file_exclude_filter=None, file_install_tag=None, - processing_optimization=False, delta_manifest_used=False) -> AnalysisResult: + processing_optimization=False) -> AnalysisResult: """ Run analysis on manifest and old manifest (if not None) and return a result with a summary resources required in order to install the provided manifest. @@ -92,7 +92,6 @@ class DLManager(Process): :param file_exclude_filter: Exclude files with this prefix from download :param file_install_tag: Only install files with the specified tag :param processing_optimization: Attempt to optimize processing order and RAM usage - :param delta_manifest_used: whether or not the manifest is a delta manifest :return: AnalysisResult """ @@ -182,10 +181,6 @@ class DLManager(Process): analysis_res.install_size = sum(fm.file_size for fm in manifest.file_manifest_list.elements if fm.filename in mc.added) - # todo properly handle delta manifests - if delta_manifest_used: - mc.removed = set() - if mc.removed: analysis_res.removed = len(mc.removed) self.log.debug(f'{analysis_res.removed} removed files') From 36d02fa5cec7a84c0b551b03e9949ed4560e5017 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 9 Sep 2020 10:28:28 +0200 Subject: [PATCH 028/101] [downloader] Fix original file offset when using delta manifests --- legendary/downloader/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 772aa02..294e17d 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -285,7 +285,7 @@ class DLManager(Process): # check if new chunk part is wholly contained in the old chunk part if cp_o <= cp.offset and (cp.offset + cp.size) <= cp_end_o: references[cp.guid_num] -= 1 - re_usable[changed][key] = file_o + re_usable[changed][key] = file_o + (cp.offset - cp_o) analysis_res.reuse_size += cp.size break From dcbf8db54d7113d835bd14eea0336372f7a7b0af Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 9 Sep 2020 10:29:46 +0200 Subject: [PATCH 029/101] [core] Properly handle Delta manifests (update base manifest) Also removes the old workaround. --- legendary/core.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index e681c0b..db4660c 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -31,6 +31,7 @@ from legendary.models.manifest import Manifest, ManifestMeta from legendary.models.chunk import Chunk from legendary.utils.game_workarounds import is_opt_enabled from legendary.utils.savegame_helper import SaveGameHelper +from legendary.utils.manifests import combine_manifests # ToDo: instead of true/false return values for success/failure actually raise an exception that the CLI/GUI @@ -652,6 +653,9 @@ class LegendaryCore: def get_delta_manifest(self, base_url, old_build_id, new_build_id): """Get optimized delta manifest (doesn't seem to exist for most games)""" + if old_build_id == new_build_id: + return None + r = self.egs.unauth_session.get(f'{base_url}/Deltas/{new_build_id}/{old_build_id}.delta') if r.status_code == 200: return r.content @@ -667,11 +671,10 @@ class LegendaryCore: file_exclude_filter: list = None, file_install_tag: list = None, dl_optimizations: bool = False, dl_timeout: int = 10, repair: bool = False, repair_use_latest: bool = False, - disable_delta: bool = False, egl_guid: str = '' - ) -> (DLManager, AnalysisResult, ManifestMeta): + disable_delta: bool = False, override_delta_manifest: str = '', + egl_guid: str = '') -> (DLManager, AnalysisResult, ManifestMeta): # load old manifest old_manifest = None - delta_manifest_used = False # load old manifest if we have one if override_old_manifest: @@ -710,19 +713,23 @@ class LegendaryCore: # save manifest with version name as well for testing/downgrading/etc. self.lgd.save_manifest(game.app_name, new_manifest_data, version=new_manifest.meta.build_version) - # also fetch optimized delta manifest (may not exist) - if old_manifest and new_manifest and not (override_old_manifest or override_manifest or disable_delta or - old_manifest.meta.build_id == new_manifest.meta.build_id): - delta_manifest_data = self.get_delta_manifest(randchoice(base_urls), - old_manifest.meta.build_id, - new_manifest.meta.build_id) + + # check if we should use a delta manifest or not + disable_delta = disable_delta or ((override_old_manifest or override_manifest) and not override_delta_manifest) + if old_manifest and new_manifest and not disable_delta: + if override_delta_manifest: + self.log.info(f'Overriding delta manifest with "{override_delta_manifest}"') + delta_manifest_data, _ = self.get_uri_manifest(override_delta_manifest) + else: + delta_manifest_data = self.get_delta_manifest(randchoice(base_urls), + old_manifest.meta.build_id, + new_manifest.meta.build_id) if delta_manifest_data: delta_manifest = self.load_manifest(delta_manifest_data) - self.log.info(f'Using optimized delta manifest to upgrade from build' - f'"{old_manifest.meta.build_id}" to' + self.log.info(f'Using optimized delta manifest to upgrade from build ' + f'"{old_manifest.meta.build_id}" to ' f'"{new_manifest.meta.build_id}"...') - new_manifest = delta_manifest - delta_manifest_used = True + combine_manifests(new_manifest, delta_manifest) else: self.log.debug(f'No Delta manifest received from CDN.') @@ -794,8 +801,7 @@ class LegendaryCore: file_prefix_filter=file_prefix_filter, file_exclude_filter=file_exclude_filter, file_install_tag=file_install_tag, - processing_optimization=process_opt, - delta_manifest_used=delta_manifest_used) + processing_optimization=process_opt) prereq = None if new_manifest.meta.prereq_ids: From f03a0a46fdcaf51f076aefaf9d08939a4979f2f4 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 9 Sep 2020 10:29:57 +0200 Subject: [PATCH 030/101] [cli] Add --delta-manifest override --- legendary/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/legendary/cli.py b/legendary/cli.py index 8b426cd..c9890bb 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -595,7 +595,8 @@ class LegendaryCLI: dl_timeout=args.dl_timeout, repair=args.repair_mode, repair_use_latest=args.repair_and_update, - disable_delta=args.disable_delta) + disable_delta=args.disable_delta, + override_delta_manifest=args.override_delta_manifest) # game is either up to date or hasn't changed, so we have nothing to do if not analysis.dl_size: @@ -1087,6 +1088,8 @@ def main(): help='Manifest URL or path to use instead of the CDN one (e.g. for downgrading)') install_parser.add_argument('--old-manifest', dest='override_old_manifest', action='store', metavar='', help='Manifest URL or path to use as the old one (e.g. for testing patching)') + install_parser.add_argument('--delta-manifest', dest='override_delta_manifest', action='store', metavar='', + help='Manifest URL or path to use as the delta one (e.g. for testing)') install_parser.add_argument('--base-url', dest='override_base_url', action='store', metavar='', help='Base URL to download from (e.g. to test or switch to a different CDNs)') install_parser.add_argument('--force', dest='force', action='store_true', From 28f0b72f422cc5a4089af623807b9450517c3f19 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 9 Sep 2020 10:32:37 +0200 Subject: [PATCH 031/101] Bump version to 0.20.1 --- legendary/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legendary/__init__.py b/legendary/__init__.py index 4dc2ee1..16d5be1 100644 --- a/legendary/__init__.py +++ b/legendary/__init__.py @@ -1,4 +1,4 @@ """Legendary!""" -__version__ = '0.20.0' +__version__ = '0.20.1' __codename__ = 'Endgame' From 7ab2c9dcc1cdf958fad7bbefe80cacaeee2086b6 Mon Sep 17 00:00:00 2001 From: Rodney Date: Wed, 30 Sep 2020 04:48:53 +0200 Subject: [PATCH 032/101] Update README.md --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index bb30303..f2a9c0d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![Discord](https://discordapp.com/api/guilds/695233346627698689/widget.png?style=shield)](https://discord.gg/UJKBwPw) [![Twitter Follow](https://img.shields.io/twitter/follow/legendary_gl?label=Follow%20us%20for%20updates%21&style=social)](https://twitter.com/legendary_gl) Legendary is an open-source game launcher that can download and install games from the Epic Games platform on Linux and Windows. -It's name as a tongue-in-cheek play on tiers of [item rarity in many MMORPGs](https://wow.gamepedia.com/Quality). +Its name as a tongue-in-cheek play on tiers of [item rarity in many MMORPGs](https://wow.gamepedia.com/Quality). Right now Legendary is in beta and not feature-complete. You might run into some bugs or issues. If you do please [create an issue on GitHub](https://github.com/derrod/legendary/issues/new) so we can fix it. @@ -87,49 +87,49 @@ Note that in this example we used `sudo` to install the package on the system, t To log in: ```` -$ legendary auth +legendary auth ```` Authentication is a little finicky since we have to go through the Epic website. The login page should open in your browser and after logging in you should be presented with a JSON response that contains a code, just copy and paste the code into your terminal to log in. On Windows you can use the `--import` flag to import the authentication from the Epic Games Launcher. Note that this will log you out of the Epic Launcher. Listing your games ```` -$ legendary list-games +legendary list-games ```` This will fetch a list of games available on your account, the first time may take a while depending on how many games you have. Installing a game ```` -$ legendary install Anemone +legendary install Anemone ```` **Important:** the name used for these commands is the app name, *not* the game's name! The app name is in the parentheses after the game title in the games list. List installed games and check for updates ```` -$ legendary list-installed --check-updates +legendary list-installed --check-updates ```` Launch (run) a game with online authentication ```` -$ legendary launch Anemone +legendary launch Anemone ```` **Tip:** most games will run fine offline (`--offline`), and thus won't require launching through legendary for online authentication. You can run `legendary launch --offline --dry-run` to get a command line that will launch the game with all parameters that would be used by the Epic Launcher. These can then be entered into any other game launcher (e.g. Lutris/Steam) if the game requires them. Importing a previously installed game ```` -$ legendary import-game Anemone /mnt/games/Epic/WorldOfGoo +legendary import-game Anemone /mnt/games/Epic/WorldOfGoo ```` **Note:** Importing will require a full verification so Legendary can correctly update the game later. Sync savegames with the Epic Cloud ```` -$ legendary sync-saves +legendary sync-saves ```` **Note:** When this command is run the first time after a supported game has been installed it will ask you to confirm or provide the path to where the savegame is located. Automatically sync all games with the Epic Games Launcher ```` -$ legendary -y egl-sync +legendary -y egl-sync ```` ## Usage From d95fd20e7616d95c8b0fb79f915aa9b97b9edcd5 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 24 Oct 2020 19:18:58 +0200 Subject: [PATCH 033/101] [core] Use non-POSIX mode for parsing manifest launch arguments (hopefully) Fixes #128 --- legendary/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legendary/core.py b/legendary/core.py index db4660c..9ad6fdc 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -336,7 +336,7 @@ class LegendaryCore: params.append(game_exe) if install.launch_parameters: - params.extend(shlex.split(install.launch_parameters)) + params.extend(shlex.split(install.launch_parameters, posix=False)) params.extend([ '-AUTH_LOGIN=unused', From d842780c73861efa312c71576f652398c3c8d432 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 24 Oct 2020 19:23:40 +0200 Subject: [PATCH 034/101] [lfs] Do not remove comments from ini files Fixes #105 --- legendary/lfs/lgndry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legendary/lfs/lgndry.py b/legendary/lfs/lgndry.py index 13cbaa2..460fc13 100644 --- a/legendary/lfs/lgndry.py +++ b/legendary/lfs/lgndry.py @@ -27,7 +27,7 @@ class LGDLFS: # EGS metadata self._game_metadata = dict() # Config with game specific settings (e.g. start parameters, env variables) - self.config = configparser.ConfigParser() + self.config = configparser.ConfigParser(comment_prefixes='/', allow_no_value=True) self.config.optionxform = str # ensure folders exist. From ace9ce8b5d6c809639d1dfc7db834bc2f8397e69 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 31 Oct 2020 19:48:08 +0100 Subject: [PATCH 035/101] [core] Show warning when looking up DLC fails Might fix #101 --- legendary/core.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/legendary/core.py b/legendary/core.py index 9ad6fdc..da0533c 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -261,6 +261,10 @@ class LegendaryCore: def get_dlc_for_game(self, app_name): game = self.get_game(app_name) + if not game: + self.log.warning(f'Metadata for {app_name} is missing!') + return [] + if game.is_dlc: # dlc shouldn't have DLC return [] From 7046b06f141765af940c2f0185f8b17762522003 Mon Sep 17 00:00:00 2001 From: derrod Date: Sun, 1 Nov 2020 14:34:36 +0100 Subject: [PATCH 036/101] [api/core/downloader] User-Agent update --- legendary/api/egs.py | 2 +- legendary/core.py | 4 ++-- legendary/downloader/workers.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/legendary/api/egs.py b/legendary/api/egs.py index e8549c2..d1893ae 100644 --- a/legendary/api/egs.py +++ b/legendary/api/egs.py @@ -10,7 +10,7 @@ from legendary.models.exceptions import InvalidCredentialsError class EPCAPI: - _user_agent = 'UELauncher/10.18.6-14188424+++Portal+Release-Live Windows/10.0.18363.1.256.64bit' + _user_agent = 'UELauncher/10.19.2-14598295+++Portal+Release-Live Windows/10.0.19041.1.256.64bit' # required for the oauth request _user_basic = '34a02cf8f4414e29b15921876da36f9a' _pw_basic = 'daafbccc737745039dffe53d94fc76cf' diff --git a/legendary/core.py b/legendary/core.py index da0533c..227178d 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -104,8 +104,8 @@ class LegendaryCore: 'X-Requested-With': 'XMLHttpRequest', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' - 'EpicGamesLauncher/10.18.6-14188424+++Portal+Release-Live ' - 'UnrealEngine/4.23.0-14188424+++Portal+Release-Live ' + 'EpicGamesLauncher/10.19.2-14598295+++Portal+Release-Live ' + 'UnrealEngine/4.23.0-14598295+++Portal+Release-Live ' 'Chrome/59.0.3071.15 Safari/537.36' }) diff --git a/legendary/downloader/workers.py b/legendary/downloader/workers.py index 3d9c558..c890a7d 100644 --- a/legendary/downloader/workers.py +++ b/legendary/downloader/workers.py @@ -22,7 +22,7 @@ class DLWorker(Process): self.o_q = out_queue self.session = requests.session() self.session.headers.update({ - 'User-Agent': 'EpicGamesLauncher/10.18.6-14188424+++Portal+Release-Live Windows/10.0.18363.1.256.64bit' + 'User-Agent': 'EpicGamesLauncher/10.19.2-14598295+++Portal+Release-Live Windows/10.0.19041.1.256.64bit' }) self.max_retries = max_retries self.shm = SharedMemory(name=shm) From 37083b01dea3a9ff3dae89c9a0faf7045ce46298 Mon Sep 17 00:00:00 2001 From: derrod Date: Sun, 1 Nov 2020 15:38:18 +0100 Subject: [PATCH 037/101] [core] Disable delta manifests if versions identical --- legendary/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/legendary/core.py b/legendary/core.py index 227178d..e89de2d 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -720,6 +720,7 @@ class LegendaryCore: # check if we should use a delta manifest or not disable_delta = disable_delta or ((override_old_manifest or override_manifest) and not override_delta_manifest) + disable_delta = disable_delta or (old_manifest.meta.build_id == new_manifest.meta.build_id) if old_manifest and new_manifest and not disable_delta: if override_delta_manifest: self.log.info(f'Overriding delta manifest with "{override_delta_manifest}"') From effc74b73b01c7c0fabfb8dc580490e49bb4ff8b Mon Sep 17 00:00:00 2001 From: derrod Date: Sun, 1 Nov 2020 16:38:43 +0100 Subject: [PATCH 038/101] [core] Exclude mods from games list --- legendary/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legendary/core.py b/legendary/core.py index e89de2d..5bb177b 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -254,7 +254,7 @@ class LegendaryCore: if game.is_dlc: _dlc[game.metadata['mainGameItem']['id']].append(game) - else: + elif not any(i['path'] == 'mods' for i in game.metadata.get('categories', [])): _ret.append(game) return _ret, _dlc From 83e3af344f6db38d349f6a4bef32f10c6d92e0a0 Mon Sep 17 00:00:00 2001 From: derrod Date: Mon, 2 Nov 2020 14:58:23 +0100 Subject: [PATCH 039/101] [core/lfs] Remove unversioned local manifest saving/loading --- legendary/core.py | 3 --- legendary/lfs/lgndry.py | 16 ++++++---------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index 5bb177b..e4d2ea3 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -713,7 +713,6 @@ class LegendaryCore: self.log.info('Parsing game manifest...') new_manifest = self.load_manifest(new_manifest_data) self.log.debug(f'Base urls: {base_urls}') - self.lgd.save_manifest(game.app_name, new_manifest_data) # save manifest with version name as well for testing/downgrading/etc. self.lgd.save_manifest(game.app_name, new_manifest_data, version=new_manifest.meta.build_version) @@ -960,7 +959,6 @@ class LegendaryCore: # parse and save manifest to disk for verification step of import new_manifest = self.load_manifest(manifest_data) - self.lgd.save_manifest(game.app_name, manifest_data) self.lgd.save_manifest(game.app_name, manifest_data, version=new_manifest.meta.build_version) install_size = sum(fm.file_size for fm in new_manifest.file_manifest_list.elements) @@ -1031,7 +1029,6 @@ class LegendaryCore: with open(manifest_filename, 'rb') as f: manifest_data = f.read() new_manifest = self.load_manifest(manifest_data) - self.lgd.save_manifest(lgd_igame.app_name, manifest_data) self.lgd.save_manifest(lgd_igame.app_name, manifest_data, version=new_manifest.meta.build_version) # mark game as installed diff --git a/legendary/lfs/lgndry.py b/legendary/lfs/lgndry.py index 460fc13..7a960f5 100644 --- a/legendary/lfs/lgndry.py +++ b/legendary/lfs/lgndry.py @@ -31,7 +31,7 @@ class LGDLFS: self.config.optionxform = str # ensure folders exist. - for f in ['', 'manifests', 'metadata', 'tmp', 'manifests/old']: + for f in ['', 'manifests', 'metadata', 'tmp']: if not os.path.exists(os.path.join(self.path, f)): os.makedirs(os.path.join(self.path, f)) @@ -123,21 +123,17 @@ class LGDLFS: open(os.path.join(self.path, 'assets.json'), 'w'), indent=2, sort_keys=True) - def _get_manifest_filename(self, app_name, version=''): - if not version: - return os.path.join(self.path, 'manifests', f'{app_name}.manifest') - else: - # if a version is specified load it from the versioned directory - fname = clean_filename(f'{app_name}_{version}') - return os.path.join(self.path, 'manifests', 'old', f'{fname}.manifest') + def _get_manifest_filename(self, app_name, version): + fname = clean_filename(f'{app_name}_{version}') + return os.path.join(self.path, 'manifests', f'{fname}.manifest') - def load_manifest(self, app_name, version=''): + def load_manifest(self, app_name, version): try: return open(self._get_manifest_filename(app_name, version), 'rb').read() except FileNotFoundError: # all other errors should propagate return None - def save_manifest(self, app_name, manifest_data, version=''): + def save_manifest(self, app_name, manifest_data, version): with open(self._get_manifest_filename(app_name, version), 'wb') as f: f.write(manifest_data) From 3ea394937bcc9ac24a30fd068bbd05e056e48b31 Mon Sep 17 00:00:00 2001 From: derrod Date: Mon, 2 Nov 2020 15:14:59 +0100 Subject: [PATCH 040/101] [lfs] Migrate old manifest structure to new --- legendary/lfs/lgndry.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/legendary/lfs/lgndry.py b/legendary/lfs/lgndry.py index 7a960f5..9f78ef6 100644 --- a/legendary/lfs/lgndry.py +++ b/legendary/lfs/lgndry.py @@ -35,6 +35,33 @@ class LGDLFS: if not os.path.exists(os.path.join(self.path, f)): os.makedirs(os.path.join(self.path, f)) + # if "old" folder exists migrate files and remove it + if os.path.exists(os.path.join(self.path, 'manifests', 'old')): + self.log.info('Migrating manifest files from old folders to new, please wait...') + # remove unversioned manifest files + for _f in os.listdir(os.path.join(self.path, 'manifests')): + if '.manifest' not in _f: + continue + if '_' not in _f or (_f.startswith('UE_') and _f.count('_') < 2): + self.log.debug(f'Deleting "{_f}" ...') + os.remove(os.path.join(self.path, 'manifests', _f)) + + # move files from "old" to the base folder + for _f in os.listdir(os.path.join(self.path, 'manifests', 'old')): + try: + self.log.debug(f'Renaming "{_f}"') + os.rename(os.path.join(self.path, 'manifests', 'old', _f), + os.path.join(self.path, 'manifests', _f)) + except Exception as e: + self.log.warning(f'Renaming manifest file "{_f}" failed: {e!r}') + + # remove "old" folder + try: + os.removedirs(os.path.join(self.path, 'manifests', 'old')) + except Exception as e: + self.log.warning(f'Removing "{os.path.join(self.path, "manifests", "old")}" folder failed: ' + f'{e!r}, please remove manually') + # try loading config self.config.read(os.path.join(self.path, 'config.ini')) # make sure "Legendary" section exists From e97941327e248611f6f765084efefe667fbda33f Mon Sep 17 00:00:00 2001 From: derrod Date: Mon, 2 Nov 2020 15:26:10 +0100 Subject: [PATCH 041/101] [core] Return empty asset list if not authenticated Fixes #106 --- legendary/core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/legendary/core.py b/legendary/core.py index e4d2ea3..9bc462e 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -197,6 +197,9 @@ class LegendaryCore: self.egs.get_game_assets(platform=platform_override)] if not self.lgd.assets or update_assets: + # if not logged in, return empty list + if not self.egs.user: + return [] self.lgd.assets = [GameAsset.from_egs_json(a) for a in self.egs.get_game_assets()] return self.lgd.assets From 477827033e80aaead163bc0f7ca9a5679576fdca Mon Sep 17 00:00:00 2001 From: derrod Date: Mon, 2 Nov 2020 15:53:11 +0100 Subject: [PATCH 042/101] [cli/lfs] Add "cleanup" command to remove unused files --- legendary/cli.py | 27 ++++++++++++++++++++++++++- legendary/lfs/lgndry.py | 22 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/legendary/cli.py b/legendary/cli.py index c9890bb..01956c6 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -1020,6 +1020,25 @@ class LegendaryCLI: print(f'EGL Sync enabled: {self.core.egl_sync_enabled}') print(f'Config directory: {self.core.lgd.path}') + def cleanup(self, args): + before = self.core.lgd.get_dir_size() + # delete metadata + logger.debug('Removing app metadata...') + app_names = set(g.app_name for g in self.core.get_assets(update_assets=False)) + self.core.lgd.clean_metadata(app_names) + + if not args.keep_manifests: + logger.debug('Removing manifests...') + installed = [(ig.app_name, ig.version) for ig in self.core.get_installed_list()] + installed.extend((ig.app_name, ig.version) for ig in self.core.get_installed_dlc_list()) + self.core.lgd.clean_manifests(installed) + + logger.debug('Removing tmp data') + self.core.lgd.clean_tmp_data() + + after = self.core.lgd.get_dir_size() + logger.info(f'Cleanup complete! Removed {(before - after)/1024/1024:.02f} MiB.') + def main(): parser = argparse.ArgumentParser(description=f'Legendary v{__version__} - "{__codename__}"') @@ -1050,6 +1069,7 @@ def main(): import_parser = subparsers.add_parser('import-game', help='Import an already installed game') egl_sync_parser = subparsers.add_parser('egl-sync', help='Setup or run Epic Games Launcher sync') status_parser = subparsers.add_parser('status', help='Show legendary status information') + clean_parser = subparsers.add_parser('cleanup', help='Remove old temporary, metadata, and manifest files') install_parser.add_argument('app_name', help='Name of the app', metavar='') uninstall_parser.add_argument('app_name', help='Name of the app', metavar='') @@ -1234,6 +1254,9 @@ def main(): status_parser.add_argument('--json', dest='json', action='store_true', help='Show status in JSON format') + clean_parser.add_argument('--keep-manifests', dest='keep_manifests', action='store_true', + help='Do not delete old manifests') + args, extra = parser.parse_known_args() if args.version: @@ -1243,7 +1266,7 @@ def main(): if args.subparser_name not in ('auth', 'list-games', 'list-installed', 'list-files', 'launch', 'download', 'uninstall', 'install', 'update', 'repair', 'list-saves', 'download-saves', 'sync-saves', - 'verify-game', 'import-game', 'egl-sync', 'status'): + 'verify-game', 'import-game', 'egl-sync', 'status', 'cleanup'): print(parser.format_help()) # Print the main help *and* the help for all of the subcommands. Thanks stackoverflow! @@ -1302,6 +1325,8 @@ def main(): cli.egs_sync(args) elif args.subparser_name == 'status': cli.status(args) + elif args.subparser_name == 'cleanup': + cli.cleanup(args) except KeyboardInterrupt: logger.info('Command was aborted via KeyboardInterrupt, cleaning up...') diff --git a/legendary/lfs/lgndry.py b/legendary/lfs/lgndry.py index 9f78ef6..1008c1e 100644 --- a/legendary/lfs/lgndry.py +++ b/legendary/lfs/lgndry.py @@ -5,6 +5,8 @@ import os import configparser import logging +from pathlib import Path + from legendary.models.game import * from legendary.utils.lfs import clean_filename @@ -195,6 +197,24 @@ class LGDLFS: except Exception as e: self.log.warning(f'Failed to delete file "{f}": {e!r}') + def clean_metadata(self, app_names): + for f in os.listdir(os.path.join(self.path, 'metadata')): + app_name = f.rpartition('.')[0] + if app_name not in app_names: + try: + os.remove(os.path.join(self.path, 'metadata', f)) + except Exception as e: + self.log.warning(f'Failed to delete file "{f}": {e!r}') + + def clean_manifests(self, in_use): + in_use_files = set(f'{clean_filename(f"{app_name}_{version}")}.manifest' for app_name, version in in_use) + for f in os.listdir(os.path.join(self.path, 'manifests')): + if f not in in_use_files: + try: + os.remove(os.path.join(self.path, 'manifests', f)) + except Exception as e: + self.log.warning(f'Failed to delete file "{f}": {e!r}') + def get_installed_game(self, app_name): if self._installed is None: try: @@ -244,3 +264,5 @@ class LGDLFS: with open(os.path.join(self.path, 'config.ini'), 'w') as cf: self.config.write(cf) + def get_dir_size(self): + return sum(f.stat().st_size for f in Path(self.path).glob('**/*') if f.is_file()) From 80841f89bb764925021b25d0eb6b61f1523fc3e1 Mon Sep 17 00:00:00 2001 From: derrod Date: Mon, 2 Nov 2020 15:55:36 +0100 Subject: [PATCH 043/101] [cli] Add "--keep-files" to uninstall without deleting --- legendary/cli.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index 01956c6..72d4594 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -748,10 +748,10 @@ class LegendaryCLI: for dlc in dlcs: if (idlc := self.core.get_installed_game(dlc.app_name)) is not None: logger.info(f'Uninstalling DLC "{dlc.app_name}"...') - self.core.uninstall_game(idlc) + self.core.uninstall_game(idlc, delete_files=not args.keep_files) logger.info(f'Removing "{igame.title}" from "{igame.install_path}"...') - self.core.uninstall_game(igame, delete_root_directory=True) + self.core.uninstall_game(igame, delete_files=not args.keep_files, delete_root_directory=True) logger.info('Game has been uninstalled.') except Exception as e: logger.warning(f'Removing game failed: {e!r}, please remove {igame.install_path} manually.') @@ -1146,6 +1146,9 @@ def main(): install_parser.add_argument('--disable-delta-manifests', dest='disable_delta', action='store_true', help='Do not use delta manfiests when updating (may increase download size)') + uninstall_parser.add_argument('--keep-files', dest='keep_files', action='store_true', + help='Keep files but remove game from Legendary database') + launch_parser.add_argument('--offline', dest='offline', action='store_true', default=False, help='Skip login and launch game without online authentication') launch_parser.add_argument('--skip-version-check', dest='skip_version_check', action='store_true', From 3310f7e7a75628508b50edef5b4f7b441b2787ed Mon Sep 17 00:00:00 2001 From: derrod Date: Mon, 2 Nov 2020 18:35:22 +0100 Subject: [PATCH 044/101] [core] Fix crash if no old manifest present --- legendary/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/legendary/core.py b/legendary/core.py index 9bc462e..0717a2e 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -722,7 +722,8 @@ class LegendaryCore: # check if we should use a delta manifest or not disable_delta = disable_delta or ((override_old_manifest or override_manifest) and not override_delta_manifest) - disable_delta = disable_delta or (old_manifest.meta.build_id == new_manifest.meta.build_id) + if old_manifest and new_manifest: + disable_delta = disable_delta or (old_manifest.meta.build_id == new_manifest.meta.build_id) if old_manifest and new_manifest and not disable_delta: if override_delta_manifest: self.log.info(f'Overriding delta manifest with "{override_delta_manifest}"') From 8206283755d692ba5d8fb6627699f49b7e69294a Mon Sep 17 00:00:00 2001 From: derrod Date: Mon, 2 Nov 2020 19:08:06 +0100 Subject: [PATCH 045/101] [core/cli] Warn/Fail if game requires Uplay Addresses #69 but does not fix it. --- legendary/cli.py | 16 +++++++++------- legendary/core.py | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index 72d4594..4093b1f 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -618,21 +618,23 @@ class LegendaryCLI: logger.info(f'Reusable size: {analysis.reuse_size / 1024 / 1024:.02f} MiB (chunks) / ' f'{analysis.unchanged / 1024 / 1024:.02f} MiB (unchanged)') - res = self.core.check_installation_conditions(analysis=analysis, install=igame, + res = self.core.check_installation_conditions(analysis=analysis, install=igame, game=game, updating=self.core.is_installed(args.app_name), ignore_space_req=args.ignore_space) - if res.failures: - logger.fatal('Download cannot proceed, the following errors occured:') - for msg in sorted(res.failures): - logger.fatal(msg) - exit(1) + if res.warnings or res.failures: + logger.info('Installation requirements check returned the following results:') if res.warnings: - logger.warning('Installation requirements check returned the following warnings:') for warn in sorted(res.warnings): logger.warning(warn) + if res.failures: + for msg in sorted(res.failures): + logger.fatal(msg) + logger.error('Installation cannot proceed, exiting.') + exit(1) + logger.info('Downloads are resumable, you can interrupt the download with ' 'CTRL-C and resume it using the same command later on.') diff --git a/legendary/core.py b/legendary/core.py index 0717a2e..79bddbf 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -833,6 +833,7 @@ class LegendaryCore: @staticmethod def check_installation_conditions(analysis: AnalysisResult, install: InstalledGame, + game: Game, updating: bool = False, ignore_space_req: bool = False) -> ConditionCheckResult: results = ConditionCheckResult(failures=set(), warnings=set()) @@ -871,6 +872,26 @@ class LegendaryCore: results.failures.add(f'Not enough available disk space!' f'{free_mib:.02f} MiB < {required_mib:.02f} MiB') + # check if the game actually ships the files or just a uplay installer + packed game files + executables = [f for f in analysis.manifest_comparison.added if + f.endswith('.exe') and not f.startswith('Installer/')] + if not any('uplay' not in e.lower() for e in executables): + results.failures.add('This game requires installation via Uplay and does not ship executable game files.') + + # check if the game launches via uplay + if install.executable == 'UplayLaunch.exe': + results.warnings.add('This game requires launching via Uplay, it is recommended to install the game ' + 'via Uplay instead.') + + # check if the game requires linking to an external account first + partner_link = game.metadata.get('customAttributes', {}).get('partnerLinkType', {}).get('value', None) + if partner_link == 'ubisoft': + results.warnings.add('This game requires linking to and activating on a Ubisoft account first, ' + 'this is not currently supported.') + elif partner_link: + results.warnings.add(f'This game requires linking to "{partner_link}", ' + f'this is currently unsupported and the game may not work.') + return results def get_default_install_dir(self): From 3e6e173772a4f548e8b0e0305535cbe8fa43dc6b Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 5 Nov 2020 15:25:37 +0100 Subject: [PATCH 046/101] [core] Fix Uplay installer check --- legendary/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index 79bddbf..11e91ef 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -874,8 +874,9 @@ class LegendaryCore: # check if the game actually ships the files or just a uplay installer + packed game files executables = [f for f in analysis.manifest_comparison.added if - f.endswith('.exe') and not f.startswith('Installer/')] - if not any('uplay' not in e.lower() for e in executables): + f.lower().endswith('.exe') and not f.startswith('Installer/')] + if not updating and not any('uplay' not in e.lower() for e in executables) and \ + any('uplay' in e.lower() for e in executables): results.failures.add('This game requires installation via Uplay and does not ship executable game files.') # check if the game launches via uplay From 3f3366c632b94423bcfc1a31f69249f6d94d8bef Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 12 Nov 2020 17:52:02 +0100 Subject: [PATCH 047/101] Bump version to 0.20.2 and Update README --- README.md | 158 +++++++++++++++++++++++++++++------------- legendary/__init__.py | 4 +- 2 files changed, 113 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index f2a9c0d..7adf3fd 100644 --- a/README.md +++ b/README.md @@ -135,18 +135,20 @@ legendary -y egl-sync ## Usage ```` -usage: legendary [-h] [-v] [-y] [-V] {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync,status} ... +usage: legendary [-h] [-v] [-y] [-V] + {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync,status,cleanup} + ... Legendary v0.X.X - "Codename" optional arguments: -h, --help show this help message and exit - -v Set loglevel to debug + -v, --debug Set loglevel to debug -y, --yes Default to yes for all prompts - -V Print version and exit + -V, --version Print version and exit Commands: - {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync,status} + {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync,status,cleanup} auth Authenticate with EPIC install (download,update,repair) Download a game @@ -162,18 +164,23 @@ Commands: import-game Import an already installed game egl-sync Setup or run Epic Games Launcher sync status Show legendary status information + cleanup Remove old temporary, metadata, and manifest files Individual command help: Command: auth -usage: legendary auth [-h] [--import] [--code ] [--sid ] [--delete] +usage: legendary auth [-h] [--import] [--code ] + [--sid ] [--delete] optional arguments: -h, --help show this help message and exit - --import Import Epic Games Launcher authentication data (logs out of EGL) + --import Import Epic Games Launcher authentication data (logs + out of EGL) --code - Use specified exchange code instead of interactive authentication - --sid Use specified session id instead of interactive authentication + Use specified exchange code instead of interactive + authentication + --sid Use specified session id instead of interactive + authentication --delete Remove existing authentication (log out) @@ -188,42 +195,64 @@ positional arguments: optional arguments: -h, --help show this help message and exit --base-path Path for game installations (defaults to ~/legendary) - --game-folder Folder for game installation (defaults to folder specified in metadata) + --game-folder Folder for game installation (defaults to folder + specified in metadata) --max-shared-memory - Maximum amount of shared memory to use (in MiB), default: 1 GiB - --max-workers Maximum amount of download workers, default: min(2 * CPUs, 16) - --manifest Manifest URL or path to use instead of the CDN one (e.g. for downgrading) - --old-manifest Manifest URL or path to use as the old one (e.g. for testing patching) - --base-url Base URL to download from (e.g. to test or switch to a different CDNs) + Maximum amount of shared memory to use (in MiB), + default: 1 GiB + --max-workers Maximum amount of download workers, default: min(2 * + CPUs, 16) + --manifest Manifest URL or path to use instead of the CDN one + (e.g. for downgrading) + --old-manifest Manifest URL or path to use as the old one (e.g. for + testing patching) + --delta-manifest + Manifest URL or path to use as the delta one (e.g. for + testing) + --base-url Base URL to download from (e.g. to test or switch to a + different CDNs) --force Download all files / ignore existing (overwrite) - --disable-patching Do not attempt to patch existing installation (download entire changed files) + --disable-patching Do not attempt to patch existing installation + (download entire changed files) --download-only, --no-install - Do not intall app and do not run prerequisite installers after download - --update-only Only update, do not do anything if specified app is not installed - --dlm-debug Set download manager and worker processes' loglevel to debug + Do not intall app and do not run prerequisite + installers after download + --update-only Only update, do not do anything if specified app is + not installed + --dlm-debug Set download manager and worker processes' loglevel to + debug --platform - Platform override for download (also sets --no-install) - --prefix Only fetch files whose path starts with (case insensitive) - --exclude Exclude files starting with (case insensitive) + Platform override for download (also sets --no- + install) + --prefix Only fetch files whose path starts with (case + insensitive) + --exclude Exclude files starting with (case + insensitive) --install-tag Only download files with the specified install tag - --enable-reordering Enable reordering optimization to reduce RAM requirements during download (may have adverse results for some titles) - --dl-timeout Connection timeout for downloader (default: 10 seconds) + --enable-reordering Enable reordering optimization to reduce RAM + requirements during download (may have adverse results + for some titles) + --dl-timeout Connection timeout for downloader (default: 10 + seconds) --save-path Set save game path to be used for sync-saves - --repair Repair installed game by checking and redownloading corrupted/missing files + --repair Repair installed game by checking and redownloading + corrupted/missing files --repair-and-update Update game to the latest version when repairing --ignore-free-space Do not abort if not enough free space is available --disable-delta-manifests - Do not use delta manfiests when updating (may increase download size) + Do not use delta manfiests when updating (may increase + download size) Command: uninstall -usage: legendary uninstall [-h] +usage: legendary uninstall [-h] [--keep-files] positional arguments: - Name of the app + Name of the app optional arguments: - -h, --help show this help message and exit + -h, --help show this help message and exit + --keep-files Keep files but remove game from Legendary database Command: launch @@ -236,16 +265,21 @@ positional arguments: optional arguments: -h, --help show this help message and exit - --offline Skip login and launch game without online authentication + --offline Skip login and launch game without online + authentication --skip-version-check Skip version check when launching game in online mode --override-username - Override username used when launching the game (only works with some titles) - --dry-run Print the command line that would have been used to launch the game and exit + Override username used when launching the game (only + works with some titles) + --dry-run Print the command line that would have been used to + launch the game and exit --language - Override language for game launch (defaults to system locale) + Override language for game launch (defaults to system + locale) --wrapper Wrapper command to launch game with - --set-defaults Save parameters used to launch to config (does not include env vars) + --set-defaults Save parameters used to launch to config (does not + include env vars) --reset-defaults Reset config settings for app and exit --wine Set WINE binary to use to launch the app --wine-prefix @@ -254,20 +288,24 @@ optional arguments: Command: list-games -usage: legendary list-games [-h] [--platform ] [--include-ue] [--csv] [--tsv] [--json] +usage: legendary list-games [-h] [--platform ] [--include-ue] [--csv] + [--tsv] [--json] optional arguments: -h, --help show this help message and exit --platform - Override platform that games are shown for (e.g. Win32/Mac) - --include-ue Also include Unreal Engine content (Engine/Marketplace) in list + Override platform that games are shown for (e.g. + Win32/Mac) + --include-ue Also include Unreal Engine content + (Engine/Marketplace) in list --csv List games in CSV format --tsv List games in TSV format --json List games in JSON format Command: list-installed -usage: legendary list-installed [-h] [--check-updates] [--csv] [--tsv] [--json] [--show-dirs] +usage: legendary list-installed [-h] [--check-updates] [--csv] [--tsv] [--json] + [--show-dirs] optional arguments: -h, --help show this help message and exit @@ -279,7 +317,10 @@ optional arguments: Command: list-files -usage: legendary list-files [-h] [--force-download] [--platform ] [--manifest ] [--csv] [--tsv] [--json] [--hashlist] [--install-tag ] [] +usage: legendary list-files [-h] [--force-download] [--platform ] + [--manifest ] [--csv] [--tsv] [--json] + [--hashlist] [--install-tag ] + [] positional arguments: Name of the app (optional) @@ -293,7 +334,8 @@ optional arguments: --csv Output in CSV format --tsv Output in TSV format --json Output in JSON format - --hashlist Output file hash list in hashcheck/sha1sum -c compatible format + --hashlist Output file hash list in hashcheck/sha1sum -c + compatible format --install-tag Show only files with specified install tag @@ -318,7 +360,10 @@ optional arguments: Command: sync-saves -usage: legendary sync-saves [-h] [--skip-upload] [--skip-download] [--force-upload] [--force-download] [--save-path ] [--disable-filters] [] +usage: legendary sync-saves [-h] [--skip-upload] [--skip-download] + [--force-upload] [--force-download] + [--save-path ] [--disable-filters] + [] positional arguments: Name of the app (optional) @@ -329,7 +374,8 @@ optional arguments: --skip-download Only upload new saves from cloud, don't download --force-upload Force upload even if local saves are older --force-download Force download even if local saves are newer - --save-path Override savegame path (requires single app name to be specified) + --save-path Override savegame path (requires single app name to be + specified) --disable-filters Disable save game file filtering @@ -344,7 +390,8 @@ optional arguments: Command: import-game -usage: legendary import-game [-h] [--disable-check] +usage: legendary import-game [-h] [--disable-check] + positional arguments: Name of the app @@ -353,24 +400,33 @@ positional arguments: optional arguments: -h, --help show this help message and exit - --disable-check Disables completeness check of the to-be-imported game installation (useful if the imported game is a much older version or missing files) + --disable-check Disables completeness check of the to-be-imported game + installation (useful if the imported game is a much + older version or missing files) Command: egl-sync -usage: legendary egl-sync [-h] [--egl-manifest-path EGL_MANIFEST_PATH] [--egl-wine-prefix EGL_WINE_PREFIX] [--enable-sync] [--disable-sync] [--one-shot] [--import-only] [--export-only] [--unlink] +usage: legendary egl-sync [-h] [--egl-manifest-path EGL_MANIFEST_PATH] + [--egl-wine-prefix EGL_WINE_PREFIX] [--enable-sync] + [--disable-sync] [--one-shot] [--import-only] + [--export-only] [--unlink] optional arguments: -h, --help show this help message and exit --egl-manifest-path EGL_MANIFEST_PATH - Path to the Epic Games Launcher's Manifests folder, should point to /ProgramData/Epic/EpicGamesLauncher/Data/Manifests + Path to the Epic Games Launcher's Manifests folder, + should point to + /ProgramData/Epic/EpicGamesLauncher/Data/Manifests --egl-wine-prefix EGL_WINE_PREFIX - Path to the WINE prefix the Epic Games Launcher is installed in + Path to the WINE prefix the Epic Games Launcher is + installed in --enable-sync Enable automatic EGL <-> Legendary sync --disable-sync Disable automatic sync and exit --one-shot Sync once, do not ask to setup automatic sync --import-only Only import games from EGL (no export) --export-only Only export games to EGL (no import) - --unlink Disable sync and remove EGL metadata from installed games + --unlink Disable sync and remove EGL metadata from installed + games Command: status @@ -380,6 +436,14 @@ optional arguments: -h, --help show this help message and exit --offline Only print offline status information, do not login --json Show status in JSON format + + +Command: cleanup +usage: legendary cleanup [-h] [--keep-manifests] + +optional arguments: + -h, --help show this help message and exit + --keep-manifests Do not delete old manifests ```` diff --git a/legendary/__init__.py b/legendary/__init__.py index 16d5be1..8e35af4 100644 --- a/legendary/__init__.py +++ b/legendary/__init__.py @@ -1,4 +1,4 @@ """Legendary!""" -__version__ = '0.20.1' -__codename__ = 'Endgame' +__version__ = '0.20.2' +__codename__ = 'Point Insertion' From 1598844bc6d66da1c70a8ea77c3fae88f498e8c1 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 21 Nov 2020 06:21:59 +0100 Subject: [PATCH 048/101] [core] Always use default.env Also fixes minor typo --- legendary/core.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index 11e91ef..c83fae9 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -382,10 +382,10 @@ class LegendaryCore: # get environment overrides from config env = os.environ.copy() + if 'default.env' in self.lgd.config: + env.update(dict(self.lgd.config['default.env'])) if f'{app_name}.env' in self.lgd.config: env.update(dict(self.lgd.config[f'{app_name}.env'])) - elif 'default.env' in self.lgd.config: - env.update(dict(self.lgd.config['default.env'])) if wine_pfx: env['WINEPREFIX'] = wine_pfx @@ -861,6 +861,7 @@ class LegendaryCore: if updating: min_disk_space += analysis.biggest_file_size + # todo when resuming, only check remaining files _, _, free = shutil.disk_usage(os.path.split(install.install_path)[0]) if free < min_disk_space: free_mib = free / 1024 / 1024 @@ -869,7 +870,7 @@ class LegendaryCore: results.warnings.add(f'Potentially not enough available disk space! ' f'{free_mib:.02f} MiB < {required_mib:.02f} MiB') else: - results.failures.add(f'Not enough available disk space!' + results.failures.add(f'Not enough available disk space! ' f'{free_mib:.02f} MiB < {required_mib:.02f} MiB') # check if the game actually ships the files or just a uplay installer + packed game files From 22b7db7a2976a5072e245cb924be8e90fe53f7c2 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 21 Nov 2020 14:30:30 +0100 Subject: [PATCH 049/101] [cli] Allow -y/--yes to be specified as part of the subcommand arguments --- legendary/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index 4093b1f..da98c8a 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -1294,10 +1294,10 @@ def main(): logging.getLogger('requests').setLevel(logging.WARNING) logging.getLogger('urllib3').setLevel(logging.WARNING) - # -y having to be specified before the subcommand is a little counter-intuitive - # For now show a warning if a user is misusing that flag + # if --yes is used as part of the subparsers arguments manually set the flag in the main parser. if '-y' in extra or '--yes' in extra: - logger.warning('-y/--yes flag needs to be specified *before* the command name') + args.yes = True + extra = [i for i in extra if i not in ('--yes', '-y')] # technically args.func() with setdefaults could work (see docs on subparsers) # but that would require all funcs to accept args and extra... From 8a98c14055c5ac1b95b834086ac4744bedbcc4f8 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 21 Nov 2020 17:57:54 +0100 Subject: [PATCH 050/101] [core] Fix CDN URI building for new Akamai CDN --- legendary/core.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index c83fae9..0c89993 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -12,7 +12,7 @@ from datetime import datetime, timezone from locale import getdefaultlocale from multiprocessing import Queue from random import choice as randchoice -from requests import Request, session +from requests import session from requests.exceptions import HTTPError from typing import List, Dict from uuid import uuid4 @@ -629,12 +629,11 @@ class LegendaryCore: if base_url not in base_urls: base_urls.append(base_url) - params = None if 'queryParams' in manifest: - params = {p['name']: p['value'] for p in manifest['queryParams']} - - # build url with a prepared request - manifest_urls.append(Request('GET', manifest['uri'], params=params).prepare().url) + params = '&'.join(f'{p["name"]}={p["value"]}' for p in manifest['queryParams']) + manifest_urls.append(f'{manifest["uri"]}?{params}') + else: + manifest_urls.append(manifest['uri']) return manifest_urls, base_urls From 81463e6908c1396c147188abc1ca556dbf705637 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 25 Nov 2020 06:53:36 +0100 Subject: [PATCH 051/101] Bump version to 0.20.3 --- legendary/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legendary/__init__.py b/legendary/__init__.py index 8e35af4..4dbbc7e 100644 --- a/legendary/__init__.py +++ b/legendary/__init__.py @@ -1,4 +1,4 @@ """Legendary!""" -__version__ = '0.20.2' +__version__ = '0.20.3' __codename__ = 'Point Insertion' From 3aeb48efdf777ec099641c36a29e14d2c4026905 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 25 Nov 2020 18:50:09 +0100 Subject: [PATCH 052/101] [core] Always initialize locale on startup --- legendary/core.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index 0c89993..1e221be 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -64,17 +64,12 @@ class LegendaryCore: self.local_timezone = datetime.now().astimezone().tzinfo self.language_code, self.country_code = ('en', 'US') - def get_locale(self): - locale = self.lgd.config.get('Legendary', 'locale', fallback=getdefaultlocale()[0]) - - if locale: + if locale := self.lgd.config.get('Legendary', 'locale', fallback=getdefaultlocale()[0]): try: self.language_code, self.country_code = locale.split('-' if '-' in locale else '_') self.log.debug(f'Set locale to {self.language_code}-{self.country_code}') - - # if egs is loaded make sure to override its language setting as well - if self.egs: - self.egs.language_code, self.egs.country_code = self.language_code, self.country_code + # adjust egs api language as well + self.egs.language_code, self.egs.country_code = self.language_code, self.country_code except Exception as e: self.log.warning(f'Getting locale failed: {e!r}, falling back to using en-US.') else: @@ -227,8 +222,6 @@ class LegendaryCore: def get_game_and_dlc_list(self, update_assets=True, platform_override=None, skip_ue=True) -> (List[Game], Dict[str, Game]): - # resolve locale - self.get_locale() _ret = [] _dlc = defaultdict(list) @@ -364,7 +357,6 @@ class LegendaryCore: language_code = self.lgd.config.get(app_name, 'language', fallback=language) if not language_code: # fall back to system or config language - self.get_locale() language_code = self.language_code params.extend([ From 5db6d9c73fcc5a54e9ad235589c0b473de17188b Mon Sep 17 00:00:00 2001 From: derrod Date: Mon, 7 Dec 2020 23:52:43 +0100 Subject: [PATCH 053/101] [core/README] Add `max_workers` config option Addresses #148 --- README.md | 2 ++ legendary/core.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/README.md b/README.md index 7adf3fd..0e136b8 100644 --- a/README.md +++ b/README.md @@ -469,6 +469,8 @@ Legendary supports some options as well as game specific configuration in `~/.co log_level = debug ; maximum shared memory (in MiB) to use for installation max_memory = 1024 +; maximum number of worker processes when downloading (fewer wokers will be slower, but also use fewer system resources) +max_workers = 8 ; default install directory install_dir = /mnt/tank/games ; locale override, must be in RFC 1766 format (e.g. "en-US") diff --git a/legendary/core.py b/legendary/core.py index 1e221be..01b47ab 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -792,6 +792,9 @@ class LegendaryCore: else: process_opt = False + if not max_workers: + max_workers = self.lgd.config.getint('Legendary', 'max_workers', fallback=0) + dlm = DLManager(install_path, base_url, resume_file=resume_file, status_q=status_q, max_shared_memory=max_shm * 1024 * 1024, max_workers=max_workers, dl_timeout=dl_timeout) From 1640a47d6a236795ee3f04b6aa60902482395f3b Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 8 Dec 2020 06:12:47 +0100 Subject: [PATCH 054/101] [downloader] Correctly support empty install tag --- legendary/downloader/manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 294e17d..44fb646 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -139,12 +139,13 @@ class DLManager(Process): # Not entirely sure what install tags are used for, only some titles have them. # Let's add it for testing anyway. - if file_install_tag: + if file_install_tag is not None: if isinstance(file_install_tag, str): file_install_tag = [file_install_tag] files_to_skip = set(i.filename for i in manifest.file_manifest_list.elements - if not any(fit in i.install_tags for fit in file_install_tag)) + if not any((fit in i.install_tags) or (not fit and not i.install_tags) + for fit in file_install_tag)) self.log.info(f'Found {len(files_to_skip)} files to skip based on install tag.') mc.added -= files_to_skip mc.changed -= files_to_skip From 80153d07b564c8af2a6387e9bae3ba0463d4ae19 Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 8 Dec 2020 06:43:43 +0100 Subject: [PATCH 055/101] [cli/utils] Add Cyberpunk 2077 language pack hack --- legendary/cli.py | 11 +++++++++ legendary/utils/game_workarounds.py | 36 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/legendary/cli.py b/legendary/cli.py index da98c8a..0fb68f5 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -24,6 +24,7 @@ from legendary.models.game import SaveGameStatus, VerifyResult from legendary.utils.cli import get_boolean_choice from legendary.utils.custom_parser import AliasedSubParsersAction from legendary.utils.lfs import validate_files +from legendary.utils.game_workarounds import cyber_prompt_2077 # todo custom formatter for cli logger (clean info, highlighted error/warning) logging.basicConfig( @@ -577,6 +578,16 @@ class LegendaryCLI: else: logger.info(f'Using existing repair file: {repair_file}') + # Workaround for Cyberpunk 2077 preload + if game.app_name.startswith('Ginger'): + if not self.core.is_installed(game.app_name): + args.install_tag = cyber_prompt_2077() + if game.app_name not in self.core.lgd.config: + self.core.lgd.config[game.app_name] = dict() + self.core.lgd.config.set(game.app_name, 'install_tags', ','.join(args.install_tag)) + else: + args.install_tag = self.core.lgd.config.get(game.app_name, 'install_tags', fallback='').split(',') + logger.info('Preparing download...') # todo use status queue to print progress from CLI # This has become a little ridiculous hasn't it? diff --git a/legendary/utils/game_workarounds.py b/legendary/utils/game_workarounds.py index 3da0d66..0ddb3b6 100644 --- a/legendary/utils/game_workarounds.py +++ b/legendary/utils/game_workarounds.py @@ -8,3 +8,39 @@ _optimize_default = { def is_opt_enabled(app_name): return app_name.lower() in _optimize_default + + +_cyberpunk_sdl = { + 'de': {'tags': ['voice_de_de'], 'name': 'Deutsch'}, + 'es': {'tags': ['voice_es_es'], 'name': 'español (España)'}, + 'fr': {'tags': ['voice_fr_fr'], 'name': 'français'}, + 'it': {'tags': ['voice_it_it'], 'name': 'italiano'}, + 'ja': {'tags': ['voice_ja_jp'], 'name': '日本語'}, + 'ko': {'tags': ['voice_ko_kr'], 'name': '한국어'}, + 'pl': {'tags': ['voice_pl_pl'], 'name': 'polski'}, + 'pt': {'tags': ['voice_pt_br'], 'name': 'português brasileiro'}, + 'ru': {'tags': ['voice_ru_ru'], 'name': 'русский'}, + 'zh': {'tags': ['voice_zh_cn'], 'name': '中文（中国）'} +} + + +def cyber_prompt_2077(): + print('You are about to install Cyberpunk 2077, this game supports selective downloads for langauge packs.') + print('The following language packs are available:') + for tag, info in _cyberpunk_sdl.items(): + print(' *', tag, '-', info['name']) + + print('Please enter a comma-separated list of language packs to install (leave blank for english only)') + choices = input('Additional languages [e.g. de,fr]: ') + if not choices: + return [''] + + tags = [''] + for c in choices.split(','): + c = c.strip() + if c in _cyberpunk_sdl: + tags.extend(_cyberpunk_sdl[c]['tags']) + else: + print('Invalid tag:', c) + + return tags From 8e012c04418b81eecc4b6d10148c5a296dc6971b Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 8 Dec 2020 23:05:52 +0100 Subject: [PATCH 056/101] Bump version to 0.20.4 --- legendary/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/legendary/__init__.py b/legendary/__init__.py index 4dbbc7e..e546575 100644 --- a/legendary/__init__.py +++ b/legendary/__init__.py @@ -1,4 +1,4 @@ """Legendary!""" -__version__ = '0.20.3' -__codename__ = 'Point Insertion' +__version__ = '0.20.4' +__codename__ = 'Cyberpunk Edition' From b7db0ac721e3b6aa146cc490bf7219ead277ab2a Mon Sep 17 00:00:00 2001 From: derrod Date: Sun, 13 Dec 2020 03:10:18 +0100 Subject: [PATCH 057/101] [api/core/downloader] Update User-Agents --- legendary/api/egs.py | 2 +- legendary/core.py | 6 +++--- legendary/downloader/workers.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/legendary/api/egs.py b/legendary/api/egs.py index d1893ae..cd73817 100644 --- a/legendary/api/egs.py +++ b/legendary/api/egs.py @@ -10,7 +10,7 @@ from legendary.models.exceptions import InvalidCredentialsError class EPCAPI: - _user_agent = 'UELauncher/10.19.2-14598295+++Portal+Release-Live Windows/10.0.19041.1.256.64bit' + _user_agent = 'UELauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit' # required for the oauth request _user_basic = '34a02cf8f4414e29b15921876da36f9a' _pw_basic = 'daafbccc737745039dffe53d94fc76cf' diff --git a/legendary/core.py b/legendary/core.py index 01b47ab..7037e45 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -99,9 +99,9 @@ class LegendaryCore: 'X-Requested-With': 'XMLHttpRequest', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' - 'EpicGamesLauncher/10.19.2-14598295+++Portal+Release-Live ' - 'UnrealEngine/4.23.0-14598295+++Portal+Release-Live ' - 'Chrome/59.0.3071.15 Safari/537.36' + 'EpicGamesLauncher/11.0.1-14907503+++Portal+Release-Live ' + 'UnrealEngine/4.23.0-14907503+++Portal+Release-Live ' + 'Chrome/84.0.4147.38 Safari/537.36' }) # get first set of cookies (EPIC_BEARER_TOKEN etc.) diff --git a/legendary/downloader/workers.py b/legendary/downloader/workers.py index c890a7d..16bde01 100644 --- a/legendary/downloader/workers.py +++ b/legendary/downloader/workers.py @@ -22,7 +22,7 @@ class DLWorker(Process): self.o_q = out_queue self.session = requests.session() self.session.headers.update({ - 'User-Agent': 'EpicGamesLauncher/10.19.2-14598295+++Portal+Release-Live Windows/10.0.19041.1.256.64bit' + 'User-Agent': 'EpicGamesLauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit' }) self.max_retries = max_retries self.shm = SharedMemory(name=shm) From 9c87f8ab4f74973a143e46417c9ecdbf616508d2 Mon Sep 17 00:00:00 2001 From: derrod Date: Mon, 14 Dec 2020 06:28:03 +0100 Subject: [PATCH 058/101] [downloader] Log optimization time and increase upper file limit --- legendary/downloader/manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 44fb646..219f322 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -195,7 +195,7 @@ class DLManager(Process): analysis_res.unchanged = len(mc.unchanged) self.log.debug(f'{analysis_res.unchanged} unchanged files') - if processing_optimization and len(manifest.file_manifest_list.elements) > 8_000: + if processing_optimization and len(manifest.file_manifest_list.elements) > 20_000: self.log.warning('Manifest contains too many files, processing optimizations will be disabled.') processing_optimization = False elif processing_optimization: @@ -221,6 +221,7 @@ class DLManager(Process): file_to_chunks[fm.filename].add(cp.guid_num) if processing_optimization: + s_time = time.time() # reorder the file manifest list to group files that share many chunks # 5 is mostly arbitrary but has shown in testing to be a good choice min_overlap = 4 @@ -265,6 +266,8 @@ class DLManager(Process): processed.add(partner) fmlist = _fmlist + opt_delta = time.time() - s_time + self.log.debug(f'Processing optimizations took {opt_delta:.01f} seconds.') # determine reusable chunks and prepare lookup table for reusable ones re_usable = defaultdict(dict) From daeee2eb8d0fb9d5e04159372df0d58316adada7 Mon Sep 17 00:00:00 2001 From: derrod Date: Mon, 14 Dec 2020 08:04:28 +0100 Subject: [PATCH 059/101] [downloader] Rework order optimizer to be significantly faster Also allow for a larger amount of files to be optimized. --- legendary/downloader/manager.py | 73 +++++++++++++-------------------- 1 file changed, 28 insertions(+), 45 deletions(-) diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 219f322..57f16b7 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -195,7 +195,7 @@ class DLManager(Process): analysis_res.unchanged = len(mc.unchanged) self.log.debug(f'{analysis_res.unchanged} unchanged files') - if processing_optimization and len(manifest.file_manifest_list.elements) > 20_000: + if processing_optimization and len(manifest.file_manifest_list.elements) > 100_000: self.log.warning('Manifest contains too many files, processing optimizations will be disabled.') processing_optimization = False elif processing_optimization: @@ -203,7 +203,6 @@ class DLManager(Process): # count references to chunks for determining runtime cache size later references = Counter() - file_to_chunks = defaultdict(set) fmlist = sorted(manifest.file_manifest_list.elements, key=lambda a: a.filename.lower()) @@ -217,53 +216,40 @@ class DLManager(Process): for cp in fm.chunk_parts: references[cp.guid_num] += 1 - if processing_optimization: - file_to_chunks[fm.filename].add(cp.guid_num) if processing_optimization: s_time = time.time() # reorder the file manifest list to group files that share many chunks - # 5 is mostly arbitrary but has shown in testing to be a good choice + # 4 is mostly arbitrary but has shown in testing to be a good choice min_overlap = 4 - # enumerate the file list to try and find a "partner" for - # each file that shares the most chunks with it. - partners = dict() - filenames = [fm.filename for fm in fmlist] + # ignore files with less than N chunk parts, this speeds things up dramatically + cp_threshold = 5 - for num, filename in enumerate(filenames[:int((len(filenames) + 1) / 2)]): - chunks = file_to_chunks[filename] - partnerlist = list() - - for other_file in filenames[num + 1:]: - overlap = len(chunks & file_to_chunks[other_file]) - if overlap > min_overlap: - partnerlist.append(other_file) - - if not partnerlist: - continue - - partners[filename] = partnerlist - - # iterate over all the files again and this time around + remaining_files = {fm.filename: {cp.guid_num for cp in fm.chunk_parts} + for fm in fmlist if fm.filename not in mc.unchanged} _fmlist = [] - processed = set() - for fm in fmlist: - if fm.filename in processed: - continue - _fmlist.append(fm) - processed.add(fm.filename) - # try to find the file's "partner" - f_partners = partners.get(fm.filename, None) - if not f_partners: - continue - # add each partner to list at this point - for partner in f_partners: - if partner in processed: - continue - partner_fm = manifest.file_manifest_list.get_file_by_path(partner) - _fmlist.append(partner_fm) - processed.add(partner) + # iterate over all files that will be downloaded and pair up those that share the most chunks + for fm in fmlist: + if fm.filename not in remaining_files: + continue + + _fmlist.append(fm) + f_chunks = remaining_files.pop(fm.filename) + if len(f_chunks) < cp_threshold: + continue + + best_overlap, match = 0, None + for fname, chunks in remaining_files.items(): + if len(chunks) < cp_threshold: + continue + overlap = len(f_chunks & chunks) + if overlap > min_overlap and overlap > best_overlap: + best_overlap, match = overlap, fname + + if match: + _fmlist.append(manifest.file_manifest_list.get_file_by_path(match)) + remaining_files.pop(match) fmlist = _fmlist opt_delta = time.time() - s_time @@ -305,10 +291,7 @@ class DLManager(Process): # runtime cache requirement by simulating adding/removing from cache during download. self.log.debug('Creating filetasks and chunktasks...') for current_file in fmlist: - # skip unchanged and empty files - if current_file.filename in mc.unchanged: - continue - elif not current_file.chunk_parts: + if not current_file.chunk_parts: self.tasks.append(FileTask(current_file.filename, empty=True)) continue From cb7ea25a185fcc8f9991d2dfce19d256fb553ec8 Mon Sep 17 00:00:00 2001 From: derrod Date: Mon, 14 Dec 2020 08:27:02 +0100 Subject: [PATCH 060/101] [core/utils] Add version check to game workarounds Also add more games to the list of optimizations being enabled by default. This will bring Pillars of Eternity down to around ~2 GiB shared memory, which is still too much but without implementing another workaround that adds a prefix filter I cannot really fix this. Certainly better than 24... --- legendary/core.py | 2 +- legendary/utils/game_workarounds.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index 7037e45..c813705 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -786,7 +786,7 @@ class LegendaryCore: if not max_shm: max_shm = self.lgd.config.getint('Legendary', 'max_memory', fallback=1024) - if dl_optimizations or is_opt_enabled(game.app_name): + if dl_optimizations or is_opt_enabled(game.app_name, new_manifest.meta.build_version): self.log.info('Download order optimizations are enabled.') process_opt = True else: diff --git a/legendary/utils/game_workarounds.py b/legendary/utils/game_workarounds.py index 0ddb3b6..ee16ec4 100644 --- a/legendary/utils/game_workarounds.py +++ b/legendary/utils/game_workarounds.py @@ -1,13 +1,22 @@ # coding: utf-8 # games where the download order optimizations are enabled by default +# a set() of versions can be specified, empty set means all versions. _optimize_default = { - 'wombat', 'snapdragon' + 'wombat': {}, # world war z + 'snapdragon': {}, # metro exodus + 'honeycreeper': {}, # diabotical + 'bcc75c246fe04e45b0c1f1c3fd52503a': { # pillars of eternity + '1.0.2' # problematic version + } } -def is_opt_enabled(app_name): - return app_name.lower() in _optimize_default +def is_opt_enabled(app_name, version): + if (versions := _optimize_default.get(app_name.lower())) is not None: + if version in versions or not versions: + return True + return False _cyberpunk_sdl = { From 5e1896cf2c9c0587616a8290690811d2ccf70a06 Mon Sep 17 00:00:00 2001 From: derrod Date: Mon, 14 Dec 2020 08:42:22 +0100 Subject: [PATCH 061/101] [downloader] Make insufficient memory message error more helpful --- legendary/downloader/manager.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 57f16b7..b8613f5 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -359,8 +359,17 @@ class DLManager(Process): if analysis_res.min_memory > self.max_shared_memory: shared_mib = f'{self.max_shared_memory / 1024 / 1024:.01f} MiB' required_mib = f'{analysis_res.min_memory / 1024 / 1024:.01f} MiB' - raise MemoryError(f'Current shared memory cache is smaller than required! {shared_mib} < {required_mib}. ' - f'Try running legendary with the --enable-reordering flag to reduce memory usage.') + suggested_mib = round(self.max_shared_memory / 1024 / 1024 + + (analysis_res.min_memory - self.max_shared_memory) / 1024 / 1024 + 32) + + if processing_optimization: + message = f'Try running legendary with "--enable-reordering --max-shared-memory {suggested_mib:.0f}"' + else: + message = 'Try running legendary with "--enable-reordering" to reduce memory usage, ' \ + f'or use "--max-shared-memory {suggested_mib:.0f}" to increase the limit.' + + raise MemoryError(f'Current shared memory cache is smaller than required: {shared_mib} < {required_mib}. ' + + message) # calculate actual dl and patch write size. analysis_res.dl_size = \ From 3e2a6011ff74259997b56d095edac3e3a7135aa7 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 16 Dec 2020 05:45:05 +0100 Subject: [PATCH 062/101] [core] Ignore comments in configuration environment variables Comments are treated as keys with no value by configparser, but env variables with None as the value are not valid so this would crash. Fixes #156 --- legendary/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index c813705..4a5cc90 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -375,9 +375,9 @@ class LegendaryCore: # get environment overrides from config env = os.environ.copy() if 'default.env' in self.lgd.config: - env.update(dict(self.lgd.config['default.env'])) + env.update({k: v for k, v in self.lgd.config[f'default.env'].items() if v and not k.startswith(';')}) if f'{app_name}.env' in self.lgd.config: - env.update(dict(self.lgd.config[f'{app_name}.env'])) + env.update({k: v for k, v in self.lgd.config[f'{app_name}.env'].items() if v and not k.startswith(';')}) if wine_pfx: env['WINEPREFIX'] = wine_pfx From 5e061d69468b6f2e7b53271fde27fac61376bb61 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 16 Dec 2020 11:18:58 +0100 Subject: [PATCH 063/101] [core/utils] Add save path resolution on Linux If a wine prefix is specified in the config, attempt to find savegames in there. --- legendary/core.py | 39 +++++++++++++++++++++++++++------ legendary/utils/wine_helpers.py | 17 ++++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 legendary/utils/wine_helpers.py diff --git a/legendary/core.py b/legendary/core.py index 4a5cc90..165cf28 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -32,6 +32,7 @@ from legendary.models.chunk import Chunk from legendary.utils.game_workarounds import is_opt_enabled from legendary.utils.savegame_helper import SaveGameHelper from legendary.utils.manifests import combine_manifests +from legendary.utils.wine_helpers import read_registry, get_shell_folders # ToDo: instead of true/false return values for success/failure actually raise an exception that the CLI/GUI @@ -412,19 +413,43 @@ class LegendaryCore: # the following variables are known: path_vars = { - '{appdata}': os.path.expandvars('%APPDATA%'), '{installdir}': igame.install_path, - '{userdir}': os.path.expandvars('%userprofile%/documents'), '{epicid}': self.lgd.userdata['account_id'] } - # the following variables are in the EGL binary but are not used by any of - # my games and I'm not sure where they actually point at: - # {UserProfile} (Probably %USERPROFILE%) - # {UserSavedGames} + + if os.name == 'nt': + path_vars.update({ + '{appdata}': os.path.expandvars('%APPDATA%'), + '{userdir}': os.path.expandvars('%userprofile%/documents'), + # '{userprofile}': os.path.expandvars('%userprofile%'), # possibly wrong + '{usersavedgames}': os.path.expandvars('%userprofile%/Saved Games') + }) + else: + # attempt to get WINE prefix from config + wine_pfx = self.lgd.config.get(app_name, 'wine_prefix', fallback=None) + if not wine_pfx: + wine_pfx = self.lgd.config.get(f'{app_name}.env', 'WINEPREFIX', fallback=None) + if not wine_pfx: + proton_pfx = self.lgd.config.get(f'{app_name}.env', 'STEAM_COMPAT_DATA_PATH', fallback=None) + if proton_pfx: + wine_pfx = f'{proton_pfx}/pfx' + + # if we have a prefix, read the `user.reg` file and get the proper paths. + if wine_pfx: + wine_reg = read_registry(wine_pfx) + wine_folders = get_shell_folders(wine_reg, wine_pfx) + # path_vars['{userprofile}'] = user_path + path_vars['{appdata}'] = wine_folders['AppData'] + # this maps to ~/Documents, but the name is locale-dependent so just resolve the symlink from WINE + path_vars['{userdir}'] = os.path.realpath(wine_folders['Personal']) + path_vars['{usersavedgames}'] = wine_folders['4C5C32FF-BB9D-43B0-B5B4-2D72E54EAAA4'] + + # replace backslashes + save_path = save_path.replace('\\', '/') # these paths should always use a forward slash new_save_path = [path_vars.get(p.lower(), p) for p in save_path.split('/')] - return os.path.join(*new_save_path) + return os.path.realpath(os.path.join(*new_save_path)) def check_savegame_state(self, path: str, save: SaveGameFile) -> (SaveGameStatus, (datetime, datetime)): latest = 0 diff --git a/legendary/utils/wine_helpers.py b/legendary/utils/wine_helpers.py new file mode 100644 index 0000000..23c54ab --- /dev/null +++ b/legendary/utils/wine_helpers.py @@ -0,0 +1,17 @@ +import os +import configparser + + +def read_registry(wine_pfx): + reg = configparser.ConfigParser(comment_prefixes=(';', '#', '/', 'WINE'), allow_no_value=True) + reg.optionxform = str + reg.read(os.path.join(wine_pfx, 'user.reg')) + return reg + + +def get_shell_folders(registry, wine_pfx): + folders = dict() + for k, v in registry['Software\\\\Microsoft\\\\Windows\\\\CurrentVersion\\\\Explorer\\\\Shell Folders'].items(): + path_cleaned = v.strip('"').strip().replace('\\\\', '/').replace('C:/', '') + folders[k.strip('"').strip()] = os.path.join(wine_pfx, 'drive_c', path_cleaned) + return folders From e710bb893f0f380feb7c934a51c71e5f3ebbe64d Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 16 Dec 2020 11:29:39 +0100 Subject: [PATCH 064/101] [core] Fall back to default wine prefix location This and the previous commit fix #41 --- legendary/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/legendary/core.py b/legendary/core.py index 165cf28..5f4b43f 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -433,9 +433,11 @@ class LegendaryCore: proton_pfx = self.lgd.config.get(f'{app_name}.env', 'STEAM_COMPAT_DATA_PATH', fallback=None) if proton_pfx: wine_pfx = f'{proton_pfx}/pfx' + if not wine_pfx: + wine_pfx = os.path.expanduser('~/.wine') # if we have a prefix, read the `user.reg` file and get the proper paths. - if wine_pfx: + if os.path.isdir(wine_pfx): wine_reg = read_registry(wine_pfx) wine_folders = get_shell_folders(wine_reg, wine_pfx) # path_vars['{userprofile}'] = user_path From bece6ef5de7158610a66ad175cdb0d29bdf20aa9 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 16 Dec 2020 12:14:31 +0100 Subject: [PATCH 065/101] [downloader] Fix skipping unneeded chunks This was erroneously removed in daeee2e --- legendary/downloader/manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index b8613f5..3434bc8 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -291,7 +291,10 @@ class DLManager(Process): # runtime cache requirement by simulating adding/removing from cache during download. self.log.debug('Creating filetasks and chunktasks...') for current_file in fmlist: - if not current_file.chunk_parts: + # skip unchanged and empty files + if current_file.filename in mc.unchanged: + continue + elif not current_file.chunk_parts: self.tasks.append(FileTask(current_file.filename, empty=True)) continue From 691fd9bc8f2277972e57367c2be8a7c8170a6777 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 16 Dec 2020 12:15:18 +0100 Subject: [PATCH 066/101] [core] Fix calculating install size (again) --- legendary/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index 5f4b43f..6e0e193 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -878,9 +878,9 @@ class LegendaryCore: results.warnings.add('This game is not marked for offline use (may still work).') # check if enough disk space is free (dl size is the approximate amount the installation will grow) - min_disk_space = analysis.uncompressed_dl_size + min_disk_space = analysis.install_size if updating: - min_disk_space += analysis.biggest_file_size + min_disk_space += analysis.biggest_file_size + analysis.install_size # todo when resuming, only check remaining files _, _, free = shutil.disk_usage(os.path.split(install.install_path)[0]) From 08c20082812b77fb687e5d89d22d1a29979b6214 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 16 Dec 2020 12:28:37 +0100 Subject: [PATCH 067/101] [downloader/models] Add silent deletion task flag --- legendary/downloader/manager.py | 2 +- legendary/downloader/workers.py | 3 ++- legendary/models/downloading.py | 6 ++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 3434bc8..8e0ab59 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -445,7 +445,7 @@ class DLManager(Process): old_filename=task.temporary_filename), timeout=1.0) elif task.delete: - self.writer_queue.put(WriterTask(task.filename, delete=True), timeout=1.0) + self.writer_queue.put(WriterTask(task.filename, delete=True, silent=task.silent), timeout=1.0) elif task.open: self.writer_queue.put(WriterTask(task.filename, fopen=True), timeout=1.0) current_file = task.filename diff --git a/legendary/downloader/workers.py b/legendary/downloader/workers.py index 16bde01..8f34e83 100644 --- a/legendary/downloader/workers.py +++ b/legendary/downloader/workers.py @@ -217,7 +217,8 @@ class FileWorker(Process): try: os.remove(full_path) except OSError as e: - logger.error(f'Removing file failed: {e!r}') + if not j.silent: + logger.error(f'Removing file failed: {e!r}') self.o_q.put(WriterTaskResult(success=True, filename=j.filename)) continue diff --git a/legendary/models/downloading.py b/legendary/models/downloading.py index 9ee9bad..7907fd4 100644 --- a/legendary/models/downloading.py +++ b/legendary/models/downloading.py @@ -28,7 +28,7 @@ class WriterTask: def __init__(self, filename, chunk_offset=0, chunk_size=0, chunk_guid=None, close=False, shared_memory=None, cache_file='', old_file='', release_memory=False, rename=False, - empty=False, kill=False, delete=False, old_filename='', fopen=False): + empty=False, kill=False, delete=False, old_filename='', fopen=False, silent=False): self.filename = filename self.empty = empty self.shm = shared_memory @@ -46,6 +46,7 @@ class WriterTask: self.rename = rename self.old_filename = old_filename + self.silent = silent # disable logging self.kill = kill # final task for worker (quit) @@ -113,7 +114,7 @@ class ChunkTask: class FileTask: def __init__(self, filename, delete=False, empty=False, fopen=False, close=False, - rename=False, temporary_filename=None): + rename=False, temporary_filename=None, silent=False): """ Download manager Task for a file @@ -130,6 +131,7 @@ class FileTask: self.close = close self.rename = rename self.temporary_filename = temporary_filename + self.silent = silent @property def is_reusing(self): From 4d138c1a2220abfa957354a3c47b374a4d8fa0f8 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 16 Dec 2020 12:29:26 +0100 Subject: [PATCH 068/101] [downloader] Silently attempt to delete files not selected for install This is to clean up when changes are made to the selected install tags --- legendary/downloader/manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 8e0ab59..86d0df6 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -137,8 +137,7 @@ class DLManager(Process): except Exception as e: self.log.warning(f'Reading resume file failed: {e!r}, continuing as normal...') - # Not entirely sure what install tags are used for, only some titles have them. - # Let's add it for testing anyway. + # Install tags are used for selective downloading, e.g. for language packs if file_install_tag is not None: if isinstance(file_install_tag, str): file_install_tag = [file_install_tag] @@ -150,6 +149,8 @@ class DLManager(Process): mc.added -= files_to_skip mc.changed -= files_to_skip mc.unchanged |= files_to_skip + for fname in sorted(files_to_skip): + self.tasks.append(FileTask(fname, delete=True, silent=True)) # if include/exclude prefix has been set: mark all files that are not to be downloaded as unchanged if file_exclude_filter: From fd004db4b933ad8b4515f57347b7bd6c17af153c Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 16 Dec 2020 12:32:19 +0100 Subject: [PATCH 069/101] [downloader] Move optional file deletion tasks to the end --- legendary/downloader/manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 86d0df6..6aa3fc6 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -138,6 +138,7 @@ class DLManager(Process): self.log.warning(f'Reading resume file failed: {e!r}, continuing as normal...') # Install tags are used for selective downloading, e.g. for language packs + additional_deletion_tasks = [] if file_install_tag is not None: if isinstance(file_install_tag, str): file_install_tag = [file_install_tag] @@ -150,7 +151,7 @@ class DLManager(Process): mc.changed -= files_to_skip mc.unchanged |= files_to_skip for fname in sorted(files_to_skip): - self.tasks.append(FileTask(fname, delete=True, silent=True)) + additional_deletion_tasks.append(FileTask(fname, delete=True, silent=True)) # if include/exclude prefix has been set: mark all files that are not to be downloaded as unchanged if file_exclude_filter: @@ -384,6 +385,7 @@ class DLManager(Process): # add jobs to remove files for fname in mc.removed: self.tasks.append(FileTask(fname, delete=True)) + self.tasks.extend(additional_deletion_tasks) analysis_res.num_chunks_cache = len(dl_cache_guids) self.chunk_data_list = manifest.chunk_data_list From 7609553b12091d9a40ec880d6e0b8cfa747288ef Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 16 Dec 2020 12:35:31 +0100 Subject: [PATCH 070/101] [core/utils] More generic/robust selective dl, add support for Fortnite Existing installations should ask for the install tags that should be used on first update. It will now be easier to add more games as well. --- legendary/cli.py | 13 ++++++----- legendary/utils/cli.py | 31 +++++++++++++++++++++++++ legendary/utils/game_workarounds.py | 36 ----------------------------- legendary/utils/selective_dl.py | 35 ++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 42 deletions(-) create mode 100644 legendary/utils/selective_dl.py diff --git a/legendary/cli.py b/legendary/cli.py index 0fb68f5..de8e434 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -21,10 +21,10 @@ from legendary import __version__, __codename__ from legendary.core import LegendaryCore from legendary.models.exceptions import InvalidCredentialsError from legendary.models.game import SaveGameStatus, VerifyResult -from legendary.utils.cli import get_boolean_choice +from legendary.utils.cli import get_boolean_choice, sdl_prompt from legendary.utils.custom_parser import AliasedSubParsersAction from legendary.utils.lfs import validate_files -from legendary.utils.game_workarounds import cyber_prompt_2077 +from legendary.utils.selective_dl import get_sdl_appname # todo custom formatter for cli logger (clean info, highlighted error/warning) logging.basicConfig( @@ -579,14 +579,15 @@ class LegendaryCLI: logger.info(f'Using existing repair file: {repair_file}') # Workaround for Cyberpunk 2077 preload - if game.app_name.startswith('Ginger'): - if not self.core.is_installed(game.app_name): - args.install_tag = cyber_prompt_2077() + if not args.install_tag and ((sdl_name := get_sdl_appname(game.app_name)) is not None): + config_tags = self.core.lgd.config.get(game.app_name, 'install_tags', fallback=None) + if not self.core.is_installed(game.app_name) or config_tags is None: + args.install_tag = sdl_prompt(sdl_name, game.app_title) if game.app_name not in self.core.lgd.config: self.core.lgd.config[game.app_name] = dict() self.core.lgd.config.set(game.app_name, 'install_tags', ','.join(args.install_tag)) else: - args.install_tag = self.core.lgd.config.get(game.app_name, 'install_tags', fallback='').split(',') + args.install_tag = config_tags.split(',') logger.info('Preparing download...') # todo use status queue to print progress from CLI diff --git a/legendary/utils/cli.py b/legendary/utils/cli.py index c044144..2ee7fcd 100644 --- a/legendary/utils/cli.py +++ b/legendary/utils/cli.py @@ -1,3 +1,6 @@ +from legendary.utils.selective_dl import games + + def get_boolean_choice(prompt, default=True): if default: yn = 'Y/n' @@ -11,3 +14,31 @@ def get_boolean_choice(prompt, default=True): return True else: return False + + +def sdl_prompt(app_name, title): + tags = [''] + if '__required' in games[app_name]: + tags.extend(games[app_name]['__required']['tags']) + + print(f'You are about to install {title}, this game supports selective downloads.') + print('The following optional packs are available:') + for tag, info in games[app_name].items(): + if tag == '__required': + continue + print(' *', tag, '-', info['name']) + + print('Please enter a comma-separated list of optional packs to install (leave blank for defaults)') + examples = ','.join([g for g in games[app_name].keys() if g != '__required'][:2]) + choices = input(f'Additional packs [e.g. {examples}]: ') + if not choices: + return tags + + for c in choices.split(','): + c = c.strip() + if c in games[app_name]: + tags.extend(games[app_name][c]['tags']) + else: + print('Invalid tag:', c) + + return tags diff --git a/legendary/utils/game_workarounds.py b/legendary/utils/game_workarounds.py index ee16ec4..ecc8d60 100644 --- a/legendary/utils/game_workarounds.py +++ b/legendary/utils/game_workarounds.py @@ -17,39 +17,3 @@ def is_opt_enabled(app_name, version): if version in versions or not versions: return True return False - - -_cyberpunk_sdl = { - 'de': {'tags': ['voice_de_de'], 'name': 'Deutsch'}, - 'es': {'tags': ['voice_es_es'], 'name': 'español (España)'}, - 'fr': {'tags': ['voice_fr_fr'], 'name': 'français'}, - 'it': {'tags': ['voice_it_it'], 'name': 'italiano'}, - 'ja': {'tags': ['voice_ja_jp'], 'name': '日本語'}, - 'ko': {'tags': ['voice_ko_kr'], 'name': '한국어'}, - 'pl': {'tags': ['voice_pl_pl'], 'name': 'polski'}, - 'pt': {'tags': ['voice_pt_br'], 'name': 'português brasileiro'}, - 'ru': {'tags': ['voice_ru_ru'], 'name': 'русский'}, - 'zh': {'tags': ['voice_zh_cn'], 'name': '中文（中国）'} -} - - -def cyber_prompt_2077(): - print('You are about to install Cyberpunk 2077, this game supports selective downloads for langauge packs.') - print('The following language packs are available:') - for tag, info in _cyberpunk_sdl.items(): - print(' *', tag, '-', info['name']) - - print('Please enter a comma-separated list of language packs to install (leave blank for english only)') - choices = input('Additional languages [e.g. de,fr]: ') - if not choices: - return [''] - - tags = [''] - for c in choices.split(','): - c = c.strip() - if c in _cyberpunk_sdl: - tags.extend(_cyberpunk_sdl[c]['tags']) - else: - print('Invalid tag:', c) - - return tags diff --git a/legendary/utils/selective_dl.py b/legendary/utils/selective_dl.py new file mode 100644 index 0000000..2a0b405 --- /dev/null +++ b/legendary/utils/selective_dl.py @@ -0,0 +1,35 @@ +_cyberpunk_sdl = { + 'de': {'tags': ['voice_de_de'], 'name': 'Deutsch'}, + 'es': {'tags': ['voice_es_es'], 'name': 'español (España)'}, + 'fr': {'tags': ['voice_fr_fr'], 'name': 'français'}, + 'it': {'tags': ['voice_it_it'], 'name': 'italiano'}, + 'ja': {'tags': ['voice_ja_jp'], 'name': '日本語'}, + 'ko': {'tags': ['voice_ko_kr'], 'name': '한국어'}, + 'pl': {'tags': ['voice_pl_pl'], 'name': 'polski'}, + 'pt': {'tags': ['voice_pt_br'], 'name': 'português brasileiro'}, + 'ru': {'tags': ['voice_ru_ru'], 'name': 'русский'}, + 'cn': {'tags': ['voice_zh_cn'], 'name': '中文（中国）'} +} + +_fortnite_sdl = { + '__required': {'tags': ['chunk0', 'chunk10'], 'name': 'Fortnite Core'}, + 'stw': {'tags': ['chunk11', 'chunk11optional'], 'name': 'Fortnite Save the World'}, + 'hd_textures': {'tags': ['chunk10optional'], 'name': 'High Resolution Textures'}, + 'lang_de': {'tags': ['chunk2'], 'name': '(Language Pack) Deutsch'}, + 'lang_fr': {'tags': ['chunk5'], 'name': '(Language Pack) français'}, + 'lang_pl': {'tags': ['chunk7'], 'name': '(Language Pack) polski'}, + 'lang_ru': {'tags': ['chunk8'], 'name': '(Language Pack) русский'}, + 'lang_cn': {'tags': ['chunk9'], 'name': '(Language Pack) 中文（中国）'} +} + +games = { + 'Fortnite': _fortnite_sdl, + 'Ginger': _cyberpunk_sdl +} + + +def get_sdl_appname(app_name): + for k in games.keys(): + if app_name.startswith(k): + return k + return None From 1430e321e6e791212714f7cbd28ac3493f262ea2 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 16 Dec 2020 12:37:12 +0100 Subject: [PATCH 071/101] [core/utils] Miscellaneous whitespace, comment, text fixes --- legendary/cli.py | 4 ++-- legendary/utils/selective_dl.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index de8e434..1c7f9b3 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -269,7 +269,7 @@ class LegendaryCLI: elif args.csv or args.tsv: writer = csv.writer(stdout, dialect='excel-tab' if args.tsv else 'excel') writer.writerow(['path', 'hash', 'size', 'install_tags']) - writer.writerows((fm.filename, fm.hash.hex(), fm.file_size, '|'.join(fm.install_tags))for fm in files) + writer.writerows((fm.filename, fm.hash.hex(), fm.file_size, '|'.join(fm.install_tags)) for fm in files) elif args.json: _files = [] for fm in files: @@ -628,7 +628,7 @@ class LegendaryCLI: logger.info(f'Download size: {analysis.dl_size / 1024 / 1024:.02f} MiB ' f'(Compression savings: {compression:.01f}%)') logger.info(f'Reusable size: {analysis.reuse_size / 1024 / 1024:.02f} MiB (chunks) / ' - f'{analysis.unchanged / 1024 / 1024:.02f} MiB (unchanged)') + f'{analysis.unchanged / 1024 / 1024:.02f} MiB (unchanged / skipped)') res = self.core.check_installation_conditions(analysis=analysis, install=igame, game=game, updating=self.core.is_installed(args.app_name), diff --git a/legendary/utils/selective_dl.py b/legendary/utils/selective_dl.py index 2a0b405..c0ec31a 100644 --- a/legendary/utils/selective_dl.py +++ b/legendary/utils/selective_dl.py @@ -1,3 +1,6 @@ +# This file contains definitions for selective downloading for supported games +# coding: utf-8 + _cyberpunk_sdl = { 'de': {'tags': ['voice_de_de'], 'name': 'Deutsch'}, 'es': {'tags': ['voice_es_es'], 'name': 'español (España)'}, From 34677cc02ec07c4d0e99bc0364a2be10f9d44837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sim=C3=A3o=20Gomes=20Viana?= Date: Wed, 16 Dec 2020 20:21:20 +0100 Subject: [PATCH 072/101] [cli/README] Fix typo in CLI help (manfiests -> manifests) (#157) --- README.md | 2 +- legendary/cli.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0e136b8..a35e47a 100644 --- a/README.md +++ b/README.md @@ -240,7 +240,7 @@ optional arguments: --repair-and-update Update game to the latest version when repairing --ignore-free-space Do not abort if not enough free space is available --disable-delta-manifests - Do not use delta manfiests when updating (may increase + Do not use delta manifests when updating (may increase download size) diff --git a/legendary/cli.py b/legendary/cli.py index 1c7f9b3..2c1dd63 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -1158,7 +1158,7 @@ def main(): install_parser.add_argument('--ignore-free-space', dest='ignore_space', action='store_true', help='Do not abort if not enough free space is available') install_parser.add_argument('--disable-delta-manifests', dest='disable_delta', action='store_true', - help='Do not use delta manfiests when updating (may increase download size)') + help='Do not use delta manifests when updating (may increase download size)') uninstall_parser.add_argument('--keep-files', dest='keep_files', action='store_true', help='Keep files but remove game from Legendary database') From df9380ab3db6ab6c7f3e5ef6742b45fa9ada6fea Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 17 Dec 2020 14:39:05 +0100 Subject: [PATCH 073/101] [core] Fix calculating install size *again* --- legendary/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legendary/core.py b/legendary/core.py index 6e0e193..e00f581 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -880,7 +880,7 @@ class LegendaryCore: # check if enough disk space is free (dl size is the approximate amount the installation will grow) min_disk_space = analysis.install_size if updating: - min_disk_space += analysis.biggest_file_size + analysis.install_size + min_disk_space += analysis.biggest_file_size # todo when resuming, only check remaining files _, _, free = shutil.disk_usage(os.path.split(install.install_path)[0]) From 30acc7d55ed99aba576d02a1551241b3f8ec1a27 Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 17 Dec 2020 14:49:56 +0100 Subject: [PATCH 074/101] [models] Add install tags to game models --- legendary/models/egl.py | 6 +++++- legendary/models/game.py | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/legendary/models/egl.py b/legendary/models/egl.py index 7b6a6f5..c125cc1 100644 --- a/legendary/models/egl.py +++ b/legendary/models/egl.py @@ -58,6 +58,7 @@ class EGLManifest: self.display_name = None self.install_location = None self.install_size = None + self.install_tags = None self.installation_guid = None self.launch_command = None self.executable = None @@ -85,6 +86,7 @@ class EGLManifest: tmp.display_name = json.pop('DisplayName', '') tmp.install_location = json.pop('InstallLocation', '') tmp.install_size = json.pop('InstallSize', 0) + tmp.install_tags = json.pop('InstallTags', []) tmp.installation_guid = json.pop('InstallationGuid', '') tmp.launch_command = json.pop('LaunchCommand', '') tmp.executable = json.pop('LaunchExecutable', '') @@ -111,6 +113,7 @@ class EGLManifest: out['DisplayName'] = self.display_name out['InstallLocation'] = self.install_location out['InstallSize'] = self.install_size + out['InstallTags'] = self.install_tags out['InstallationGuid'] = self.installation_guid out['LaunchCommand'] = self.launch_command out['LaunchExecutable'] = self.executable @@ -136,6 +139,7 @@ class EGLManifest: tmp.display_name = igame.title tmp.install_location = igame.install_path tmp.install_size = igame.install_size + tmp.install_tags = igame.install_tags tmp.installation_guid = igame.egl_guid tmp.launch_command = igame.launch_parameters tmp.executable = igame.executable @@ -155,4 +159,4 @@ class EGLManifest: launch_parameters=self.launch_command, can_run_offline=self.can_run_offline, requires_ot=self.ownership_token, is_dlc=False, needs_verification=self.needs_validation, install_size=self.install_size, - egl_guid=self.installation_guid) + egl_guid=self.installation_guid, install_tags=self.install_tags) diff --git a/legendary/models/game.py b/legendary/models/game.py index 1e8c86c..16e2d79 100644 --- a/legendary/models/game.py +++ b/legendary/models/game.py @@ -79,7 +79,7 @@ class InstalledGame: def __init__(self, app_name='', title='', version='', manifest_path='', base_urls=None, install_path='', executable='', launch_parameters='', prereq_info=None, can_run_offline=False, requires_ot=False, is_dlc=False, save_path=None, - needs_verification=False, install_size=0, egl_guid=''): + needs_verification=False, install_size=0, egl_guid='', install_tags=None): self.app_name = app_name self.title = title self.version = version @@ -97,6 +97,7 @@ class InstalledGame: self.needs_verification = needs_verification self.install_size = install_size self.egl_guid = egl_guid + self.install_tags = install_tags if install_tags else [] @classmethod def from_json(cls, json): @@ -119,6 +120,7 @@ class InstalledGame: tmp.needs_verification = json.get('needs_verification', False) is True tmp.install_size = json.get('install_size', 0) tmp.egl_guid = json.get('egl_guid', '') + tmp.install_tags = json.get('install_tags', []) return tmp From cff8abd0dac8629b8508c148a167b4ef7e763ea6 Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 17 Dec 2020 14:54:31 +0100 Subject: [PATCH 075/101] [core] Import/Export install tags to EGL --- legendary/core.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index e00f581..836702d 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -1079,6 +1079,13 @@ class LegendaryCore: new_manifest = self.load_manifest(manifest_data) self.lgd.save_manifest(lgd_igame.app_name, manifest_data, version=new_manifest.meta.build_version) + + # transfer install tag choices to config + if lgd_igame.install_tags: + if app_name not in self.lgd.config: + self.lgd.config[app_name] = dict() + self.lgd.config.set(app_name, 'install_tags', ','.join(lgd_igame.install_tags)) + # mark game as installed _ = self._install_game(lgd_igame) return @@ -1153,7 +1160,8 @@ class LegendaryCore: return self.egl_restore_or_uninstall(lgd_igame) else: egl_igame = self.egl.get_manifest(app_name) - if egl_igame.app_version_string != lgd_igame.version: + if (egl_igame.app_version_string != lgd_igame.version) or \ + (egl_igame.install_tags != lgd_igame.install_tags): self.log.info(f'App "{egl_igame.app_name}" has been updated from EGL, syncing...') return self.egl_import(egl_igame.app_name) else: @@ -1168,7 +1176,8 @@ class LegendaryCore: self.egl_import(egl_igame.app_name) else: lgd_igame = self._get_installed_game(egl_igame.app_name) - if lgd_igame.version != egl_igame.app_version_string: + if (egl_igame.app_version_string != lgd_igame.version) or \ + (egl_igame.install_tags != lgd_igame.install_tags): self.log.info(f'App "{egl_igame.app_name}" has been updated from EGL, syncing...') self.egl_import(egl_igame.app_name) From 86ea066e8d93635d4881616970105cabb4b62102 Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 17 Dec 2020 14:54:47 +0100 Subject: [PATCH 076/101] [core] Save install tags to InstalledGame --- legendary/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legendary/core.py b/legendary/core.py index 836702d..eaac509 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -847,7 +847,7 @@ class LegendaryCore: launch_parameters=new_manifest.meta.launch_command, can_run_offline=offline == 'true', requires_ot=ot == 'true', is_dlc=base_game is not None, install_size=anlres.install_size, - egl_guid=egl_guid) + egl_guid=egl_guid, install_tags=file_install_tag) return dlm, anlres, igame From dca216d053dc3b43ee1f9f245fda06454b139b42 Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 17 Dec 2020 14:55:32 +0100 Subject: [PATCH 077/101] [cli] Fix repair-and-update not working correctly --- legendary/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legendary/cli.py b/legendary/cli.py index 2c1dd63..18bd57d 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -524,7 +524,7 @@ class LegendaryCLI: args.no_install = True elif args.subparser_name == 'repair' or args.repair_mode: args.repair_mode = True - args.no_install = True + args.no_install = args.repair_and_update is False repair_file = os.path.join(self.core.lgd.get_tmp_path(), f'{args.app_name}.repair') if not self.core.login(): From efad90d951cdee8f02a6362f72b3c8ac47ac0def Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 17 Dec 2020 14:55:56 +0100 Subject: [PATCH 078/101] [cli] Do not run SDL for DLC --- legendary/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legendary/cli.py b/legendary/cli.py index 18bd57d..da91535 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -579,7 +579,7 @@ class LegendaryCLI: logger.info(f'Using existing repair file: {repair_file}') # Workaround for Cyberpunk 2077 preload - if not args.install_tag and ((sdl_name := get_sdl_appname(game.app_name)) is not None): + if not args.install_tag and not game.is_dlc and ((sdl_name := get_sdl_appname(game.app_name)) is not None): config_tags = self.core.lgd.config.get(game.app_name, 'install_tags', fallback=None) if not self.core.is_installed(game.app_name) or config_tags is None: args.install_tag = sdl_prompt(sdl_name, game.app_title) From 51c8b67f91de4bb33ac5990f657ecaadec4e2f77 Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 17 Dec 2020 14:56:09 +0100 Subject: [PATCH 079/101] [cli] Add flag to reset SDL selection --- legendary/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/legendary/cli.py b/legendary/cli.py index da91535..d11f10d 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -581,7 +581,7 @@ class LegendaryCLI: # Workaround for Cyberpunk 2077 preload if not args.install_tag and not game.is_dlc and ((sdl_name := get_sdl_appname(game.app_name)) is not None): config_tags = self.core.lgd.config.get(game.app_name, 'install_tags', fallback=None) - if not self.core.is_installed(game.app_name) or config_tags is None: + if not self.core.is_installed(game.app_name) or config_tags is None or args.reset_sdl: args.install_tag = sdl_prompt(sdl_name, game.app_title) if game.app_name not in self.core.lgd.config: self.core.lgd.config[game.app_name] = dict() @@ -1159,6 +1159,8 @@ def main(): help='Do not abort if not enough free space is available') install_parser.add_argument('--disable-delta-manifests', dest='disable_delta', action='store_true', help='Do not use delta manifests when updating (may increase download size)') + install_parser.add_argument('--reset-sdl', dest='reset_sdl', action='store_true', + help='Reset selective downloading choices (requires repair to download new components)') uninstall_parser.add_argument('--keep-files', dest='keep_files', action='store_true', help='Keep files but remove game from Legendary database') From 40f8c553bad2557cc7ec07787e50aa429bea1946 Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 17 Dec 2020 15:21:29 +0100 Subject: [PATCH 080/101] [utils] Allow silent deletion of files --- legendary/utils/lfs.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/legendary/utils/lfs.py b/legendary/utils/lfs.py index c549cff..f9accac 100644 --- a/legendary/utils/lfs.py +++ b/legendary/utils/lfs.py @@ -27,7 +27,8 @@ def delete_folder(path: str, recursive=True) -> bool: def delete_filelist(path: str, filenames: List[str], - delete_root_directory: bool = False) -> bool: + delete_root_directory: bool = False, + silent: bool = False) -> bool: dirs = set() no_error = True @@ -40,7 +41,8 @@ def delete_filelist(path: str, filenames: List[str], try: os.remove(os.path.join(path, _dir, _fn)) except Exception as e: - logger.error(f'Failed deleting file {filename} with {e!r}') + if not silent: + logger.error(f'Failed deleting file {filename} with {e!r}') no_error = False # add intermediate directories that would have been missed otherwise @@ -58,14 +60,16 @@ def delete_filelist(path: str, filenames: List[str], # directory has already been deleted, ignore that continue except Exception as e: - logger.error(f'Failed removing directory "{_dir}" with {e!r}') + if not silent: + logger.error(f'Failed removing directory "{_dir}" with {e!r}') no_error = False if delete_root_directory: try: os.rmdir(path) except Exception as e: - logger.error(f'Removing game directory failed with {e!r}') + if not silent: + logger.error(f'Removing game directory failed with {e!r}') return no_error From 1cec4c6cb02f6d6da5336045f3601f48071ec814 Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 17 Dec 2020 15:21:51 +0100 Subject: [PATCH 081/101] [core] Add method to remove untagged files --- legendary/core.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/legendary/core.py b/legendary/core.py index eaac509..e5314b9 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -954,6 +954,18 @@ class LegendaryCore: self.lgd.remove_installed_game(installed_game.app_name) + def uninstall_tag(self, installed_game: InstalledGame): + manifest = self.load_manifest(self.get_installed_manifest(installed_game.app_name)[0]) + tags = installed_game.install_tags + if '' not in tags: + tags.append('') + + filelist = [i.filename for i in manifest.file_manifest_list.elements if not any( + (fit in i.install_tags) or (not fit and not i.install_tags) for fit in tags + )] + if not delete_filelist(installed_game.install_path, filelist, silent=True): + self.log.debug(f'Deleting some deselected files failed, but that\'s okay.') + def prereq_installed(self, app_name): igame = self.lgd.get_installed_game(app_name) igame.prereq_info['installed'] = True From 1226cd1b08b76d0e7e4520b29109e873f0a1677e Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 17 Dec 2020 15:22:26 +0100 Subject: [PATCH 082/101] [cli] Remove untagged files when install tags changed --- legendary/cli.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index d11f10d..73f76dd 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -612,15 +612,23 @@ class LegendaryCLI: # game is either up to date or hasn't changed, so we have nothing to do if not analysis.dl_size: + old_igame = self.core.get_installed_game(game.app_name) logger.info('Download size is 0, the game is either already up to date or has not changed. Exiting...') if args.repair_mode and os.path.exists(repair_file): - igame = self.core.get_installed_game(game.app_name) - if igame.needs_verification: - igame.needs_verification = False - self.core.install_game(igame) + if old_igame.needs_verification: + old_igame.needs_verification = False + self.core.install_game(old_igame) logger.debug('Removing repair file.') os.remove(repair_file) + + # check if install tags have changed, if they did; try deleting files that are no longer required. + if old_igame.install_tags != igame.install_tags: + old_igame.install_tags = igame.install_tags + self.logger.info('Deleting now untagged files.') + self.core.uninstall_tag(old_igame) + self.core.install_game(old_igame) + exit(0) logger.info(f'Install size: {analysis.install_size / 1024 / 1024:.02f} MiB') @@ -708,15 +716,22 @@ class LegendaryCLI: logger.info('This game supports cloud saves, syncing is handled by the "sync-saves" command.') logger.info(f'To download saves for this game run "legendary sync-saves {args.app_name}"') + old_igame = self.core.get_installed_game(game.app_name) if args.repair_mode and os.path.exists(repair_file): - igame = self.core.get_installed_game(game.app_name) - if igame.needs_verification: - igame.needs_verification = False - self.core.install_game(igame) + if old_igame.needs_verification: + old_igame.needs_verification = False + self.core.install_game(old_igame) logger.debug('Removing repair file.') os.remove(repair_file) + # check if install tags have changed, if they did; try deleting files that are no longer required. + if old_igame.install_tags != igame.install_tags: + old_igame.install_tags = igame.install_tags + self.logger.info('Deleting now untagged files.') + self.core.uninstall_tag(old_igame) + self.core.install_game(old_igame) + logger.info(f'Finished installation process in {end_t - start_t:.02f} seconds.') def _handle_postinstall(self, postinstall, igame, yes=False): From f42d63767c687954db7e077a48549cb519ae4553 Mon Sep 17 00:00:00 2001 From: derrod Date: Thu, 17 Dec 2020 16:31:38 +0100 Subject: [PATCH 083/101] [cli] Fix crash when not installing --- legendary/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index 73f76dd..527cf52 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -614,7 +614,7 @@ class LegendaryCLI: if not analysis.dl_size: old_igame = self.core.get_installed_game(game.app_name) logger.info('Download size is 0, the game is either already up to date or has not changed. Exiting...') - if args.repair_mode and os.path.exists(repair_file): + if old_igame and args.repair_mode and os.path.exists(repair_file): if old_igame.needs_verification: old_igame.needs_verification = False self.core.install_game(old_igame) @@ -623,7 +623,7 @@ class LegendaryCLI: os.remove(repair_file) # check if install tags have changed, if they did; try deleting files that are no longer required. - if old_igame.install_tags != igame.install_tags: + if old_igame and old_igame.install_tags != igame.install_tags: old_igame.install_tags = igame.install_tags self.logger.info('Deleting now untagged files.') self.core.uninstall_tag(old_igame) @@ -717,7 +717,7 @@ class LegendaryCLI: logger.info(f'To download saves for this game run "legendary sync-saves {args.app_name}"') old_igame = self.core.get_installed_game(game.app_name) - if args.repair_mode and os.path.exists(repair_file): + if old_igame and args.repair_mode and os.path.exists(repair_file): if old_igame.needs_verification: old_igame.needs_verification = False self.core.install_game(old_igame) @@ -726,7 +726,7 @@ class LegendaryCLI: os.remove(repair_file) # check if install tags have changed, if they did; try deleting files that are no longer required. - if old_igame.install_tags != igame.install_tags: + if old_igame and old_igame.install_tags != igame.install_tags: old_igame.install_tags = igame.install_tags self.logger.info('Deleting now untagged files.') self.core.uninstall_tag(old_igame) From 1a015a3ba71b7c730aa43feb5937876ade109b2d Mon Sep 17 00:00:00 2001 From: derrod Date: Fri, 18 Dec 2020 17:01:52 +0100 Subject: [PATCH 084/101] [core] Only attempt to delete untagged files that exist --- legendary/core.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/legendary/core.py b/legendary/core.py index e5314b9..11cf1a7 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -960,11 +960,15 @@ class LegendaryCore: if '' not in tags: tags.append('') - filelist = [i.filename for i in manifest.file_manifest_list.elements if not any( - (fit in i.install_tags) or (not fit and not i.install_tags) for fit in tags - )] - if not delete_filelist(installed_game.install_path, filelist, silent=True): - self.log.debug(f'Deleting some deselected files failed, but that\'s okay.') + # Create list of files that are now no longer needed *and* actually exist on disk + filelist = [ + fm.filename for fm in manifest.file_manifest_list.elements if + not any(((fit in fm.install_tags) or (not fit and not fm.install_tags)) for fit in tags) + and os.path.exists(os.path.join(installed_game.install_path, fm.filename)) + ] + + if not delete_filelist(installed_game.install_path, filelist): + self.log.warning(f'Deleting some deselected files failed, please check/remove manually.') def prereq_installed(self, app_name): igame = self.lgd.get_installed_game(app_name) From 254c22eaec77a3aab7858f193aaac7dab779d5c9 Mon Sep 17 00:00:00 2001 From: derrod Date: Fri, 18 Dec 2020 17:02:00 +0100 Subject: [PATCH 085/101] Update README & Bump version --- README.md | 32 +++++++++++++++++--------------- legendary/__init__.py | 4 ++-- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index a35e47a..749d82a 100644 --- a/README.md +++ b/README.md @@ -136,8 +136,8 @@ legendary -y egl-sync ```` usage: legendary [-h] [-v] [-y] [-V] - {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync,status,cleanup} - ... + {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync,status,cleanup} + ... Legendary v0.X.X - "Codename" @@ -170,7 +170,7 @@ Individual command help: Command: auth usage: legendary auth [-h] [--import] [--code ] - [--sid ] [--delete] + [--sid ] [--delete] optional arguments: -h, --help show this help message and exit @@ -242,6 +242,8 @@ optional arguments: --disable-delta-manifests Do not use delta manifests when updating (may increase download size) + --reset-sdl Reset selective downloading choices (requires repair + to download new components) Command: uninstall @@ -289,7 +291,7 @@ optional arguments: Command: list-games usage: legendary list-games [-h] [--platform ] [--include-ue] [--csv] - [--tsv] [--json] + [--tsv] [--json] optional arguments: -h, --help show this help message and exit @@ -305,7 +307,7 @@ optional arguments: Command: list-installed usage: legendary list-installed [-h] [--check-updates] [--csv] [--tsv] [--json] - [--show-dirs] + [--show-dirs] optional arguments: -h, --help show this help message and exit @@ -318,9 +320,9 @@ optional arguments: Command: list-files usage: legendary list-files [-h] [--force-download] [--platform ] - [--manifest ] [--csv] [--tsv] [--json] - [--hashlist] [--install-tag ] - [] + [--manifest ] [--csv] [--tsv] [--json] + [--hashlist] [--install-tag ] + [] positional arguments: Name of the app (optional) @@ -361,9 +363,9 @@ optional arguments: Command: sync-saves usage: legendary sync-saves [-h] [--skip-upload] [--skip-download] - [--force-upload] [--force-download] - [--save-path ] [--disable-filters] - [] + [--force-upload] [--force-download] + [--save-path ] [--disable-filters] + [] positional arguments: Name of the app (optional) @@ -391,7 +393,7 @@ optional arguments: Command: import-game usage: legendary import-game [-h] [--disable-check] - + positional arguments: Name of the app @@ -407,9 +409,9 @@ optional arguments: Command: egl-sync usage: legendary egl-sync [-h] [--egl-manifest-path EGL_MANIFEST_PATH] - [--egl-wine-prefix EGL_WINE_PREFIX] [--enable-sync] - [--disable-sync] [--one-shot] [--import-only] - [--export-only] [--unlink] + [--egl-wine-prefix EGL_WINE_PREFIX] [--enable-sync] + [--disable-sync] [--one-shot] [--import-only] + [--export-only] [--unlink] optional arguments: -h, --help show this help message and exit diff --git a/legendary/__init__.py b/legendary/__init__.py index e546575..e281b71 100644 --- a/legendary/__init__.py +++ b/legendary/__init__.py @@ -1,4 +1,4 @@ """Legendary!""" -__version__ = '0.20.4' -__codename__ = 'Cyberpunk Edition' +__version__ = '0.20.5.rc1' +__codename__ = 'A Red Letter Day (RC1)' From 104b928e3aa5feafc5e9b3e20360a7b62ca60c82 Mon Sep 17 00:00:00 2001 From: derrod Date: Fri, 18 Dec 2020 17:26:51 +0100 Subject: [PATCH 086/101] [core] Increase default shared memory to 2048 MiB This should still be reasonable for most machines people are going to run games on, and means no manual increases are required for every game it has been previously needed for. --- legendary/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legendary/core.py b/legendary/core.py index 11cf1a7..2fccb36 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -811,7 +811,7 @@ class LegendaryCore: self.log.debug(f'Using base URL: {base_url}') if not max_shm: - max_shm = self.lgd.config.getint('Legendary', 'max_memory', fallback=1024) + max_shm = self.lgd.config.getint('Legendary', 'max_memory', fallback=2048) if dl_optimizations or is_opt_enabled(game.app_name, new_manifest.meta.build_version): self.log.info('Download order optimizations are enabled.') From bd0b9248ee5a1375691e169ef75d3322aa6715cd Mon Sep 17 00:00:00 2001 From: derrod Date: Sun, 20 Dec 2020 20:48:31 +0100 Subject: [PATCH 087/101] Bump version to 0.20.5 --- legendary/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/legendary/__init__.py b/legendary/__init__.py index e281b71..fa0464a 100644 --- a/legendary/__init__.py +++ b/legendary/__init__.py @@ -1,4 +1,4 @@ """Legendary!""" -__version__ = '0.20.5.rc1' -__codename__ = 'A Red Letter Day (RC1)' +__version__ = '0.20.5' +__codename__ = 'A Red Letter Day' From d861668bc1dfe7098ceca0cfd7db44eb7c267987 Mon Sep 17 00:00:00 2001 From: derrod Date: Tue, 22 Dec 2020 18:10:03 +0100 Subject: [PATCH 088/101] [models] Update CustomFields with items() method --- legendary/models/manifest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/legendary/models/manifest.py b/legendary/models/manifest.py index 2f88796..89b0d9c 100644 --- a/legendary/models/manifest.py +++ b/legendary/models/manifest.py @@ -627,7 +627,7 @@ class ChunkPart: guid_readable, self.offset, self.size) -class CustomFields: # this could probably be replaced with just a dict +class CustomFields: def __init__(self): self.size = 0 self.version = 0 @@ -644,6 +644,9 @@ class CustomFields: # this could probably be replaced with just a dict def __str__(self): return str(self._dict) + def items(self): + return self._dict.items() + def keys(self): return self._dict.keys() From ba867f1ce64f81e566b53a536d8658aa009d62a0 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 23 Dec 2020 18:04:53 +0100 Subject: [PATCH 089/101] [cli] Sort game names case-insensitively --- legendary/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index 527cf52..999492c 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -147,9 +147,9 @@ class LegendaryCLI: platform_override=args.platform_override, skip_ue=not args.include_ue ) # sort games and dlc by name - games = sorted(games, key=lambda x: x.app_title) + games = sorted(games, key=lambda x: x.app_title.lower()) for citem_id in dlc_list.keys(): - dlc_list[citem_id] = sorted(dlc_list[citem_id], key=lambda d: d.app_title) + dlc_list[citem_id] = sorted(dlc_list[citem_id], key=lambda d: d.app_title.lower()) if args.csv or args.tsv: writer = csv.writer(stdout, dialect='excel-tab' if args.tsv else 'excel') @@ -187,7 +187,7 @@ class LegendaryCLI: self.core.get_assets(True) games = sorted(self.core.get_installed_list(), - key=lambda x: x.title) + key=lambda x: x.title.lower()) versions = dict() for game in games: From 3c71229b7ee4ec3230661f412a7ba133c83b5a79 Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 26 Dec 2020 22:01:30 +0100 Subject: [PATCH 090/101] [core] Fix missing {} around KnownFolders GUID --- legendary/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legendary/core.py b/legendary/core.py index 2fccb36..9b0225a 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -444,7 +444,7 @@ class LegendaryCore: path_vars['{appdata}'] = wine_folders['AppData'] # this maps to ~/Documents, but the name is locale-dependent so just resolve the symlink from WINE path_vars['{userdir}'] = os.path.realpath(wine_folders['Personal']) - path_vars['{usersavedgames}'] = wine_folders['4C5C32FF-BB9D-43B0-B5B4-2D72E54EAAA4'] + path_vars['{usersavedgames}'] = wine_folders['{4C5C32FF-BB9D-43B0-B5B4-2D72E54EAAA4}'] # replace backslashes save_path = save_path.replace('\\', '/') From 01a140d3565257f6ad7a8f796554510e25f8690d Mon Sep 17 00:00:00 2001 From: derrod Date: Sun, 27 Dec 2020 22:01:23 +0100 Subject: [PATCH 091/101] Update README --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 749d82a..e8f34c0 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,9 @@ Its name as a tongue-in-cheek play on tiers of [item rarity in many MMORPGs](htt Right now Legendary is in beta and not feature-complete. You might run into some bugs or issues. If you do please [create an issue on GitHub](https://github.com/derrod/legendary/issues/new) so we can fix it. +**Note:** Legendary is currently a CLI (command-line interface) application without a graphical user interface, +it has to be run from a terminal (e.g. PowerShell) + **What works:** - Authenticating with Epic's service - Downloading and installing your games and their DLC @@ -85,6 +88,8 @@ Note that in this example we used `sudo` to install the package on the system, t ## Quickstart +**Tip:** When using PowerShell with a standalone executable all commands have to be prefixed with `.\` + To log in: ```` legendary auth From 19c66bee114ad3d234c0447fc7347c03be0f7def Mon Sep 17 00:00:00 2001 From: derrod Date: Sun, 27 Dec 2020 22:01:45 +0100 Subject: [PATCH 092/101] Bump version to 0.20.6 --- legendary/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legendary/__init__.py b/legendary/__init__.py index fa0464a..1760b2a 100644 --- a/legendary/__init__.py +++ b/legendary/__init__.py @@ -1,4 +1,4 @@ """Legendary!""" -__version__ = '0.20.5' +__version__ = '0.20.6' __codename__ = 'A Red Letter Day' From a25307da71f6d074d276c369360ac48e280d18d9 Mon Sep 17 00:00:00 2001 From: TheGreatCabbage <20987172+TheGreatCabbage@users.noreply.github.com> Date: Tue, 29 Dec 2020 17:18:17 +0000 Subject: [PATCH 093/101] Update installation instructions (#177) --- README.md | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e8f34c0..c5fd0b5 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ Note that since packages are maintained by third parties it may take a bit for t If you always want to have the latest features and fixes available then using the PyPI distribution is recommended. ### Standalone + Download the `legendary` or `legendary.exe` binary from [the latest release](https://github.com/derrod/legendary/releases/latest) and move it to somewhere in your `$PATH`/`%PATH%`. Don't forget to `chmod +x` it on Linux. @@ -58,15 +59,25 @@ Note that on Linux glibc >= 2.25 is required, so older distributions such as Ubu ### Python package +#### Prerequisites + +To prevent problems with permissions during installation, please upgrade your `pip` by running `python -m pip install -U pip --user`. + +> **Tip:** You may need to replace `python` in the above command with `python3.8` on Linux, or `py -3.8` on Windows. + +#### Installation from PyPI (recommended) + Legendary is available on [PyPI](https://pypi.org/project/legendary-gl/), to install simply run: + ```bash pip install legendary-gl ``` #### Manually from the repo + - Install python3.8, setuptools, wheel, and requests - Clone the git repository and cd into it -- Run `python3.8 setup.py install` +- Run `pip install .` #### Ubuntu 20.04 example @@ -75,20 +86,26 @@ Ubuntu 20.04's standard repositories include everything needed to install legend sudo apt install python3 python3-requests python3-setuptools-git git clone https://github.com/derrod/legendary.git cd legendary -sudo python3 setup.py install +pip install . ```` -Note that in this example we used `sudo` to install the package on the system, this may not be advisable depending on your setup. +If the `legendary` executable is not available after installation, you may need to configure your `PATH` correctly. You can do this by running the command: + +```bash +echo 'export PATH=$PATH:~/.local/bin' >> ~/.profile && source ~/.profile +``` ### Directly from the repo (for dev/testing) - Install python3.8 and requests (optionally in a venv) -- cd into `legendary/` (the folder with `cli.py`) -- run `PYTHONPATH=.. python3.8 cli.py` +- cd into the repository +- Run `pip install -e .` + +This installs `legendary` in "editable" mode - any changes to the source code will take effect next time the `legendary` executable runs. ## Quickstart -**Tip:** When using PowerShell with a standalone executable all commands have to be prefixed with `.\` +**Tip:** When using PowerShell with the standalone executable, you may need to replace `legendary` with `.\legendary` in the commands below. To log in: ```` From 58edb22fcebc82fb644a00f7124face3f23e387c Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 2 Jan 2021 06:31:47 +0100 Subject: [PATCH 094/101] [cli] egl-sync: Error out if asset information missing --- legendary/cli.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/legendary/cli.py b/legendary/cli.py index 999492c..de1345d 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -911,6 +911,11 @@ class LegendaryCLI: self.core.lgd.config.remove_option('Legendary', 'egl_sync') return + if not self.core.lgd.assets: + logger.error('Legendary is missing game metadata, please login (if not already) and use the ' + '"status" command to fetch necessary information to set-up syncing.') + return + if not self.core.egl.programdata_path: if not args.egl_manifest_path and not args.egl_wine_prefix: # search default Lutris install path From ee3b3fb9fead8ee9221fa46f9791d46b9477e8eb Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 2 Jan 2021 06:53:21 +0100 Subject: [PATCH 095/101] Fix various typos and some style complaints Closes #175 --- README.md | 4 ++-- legendary/cli.py | 6 +++--- legendary/core.py | 5 ++--- legendary/downloader/manager.py | 2 +- legendary/models/downloading.py | 2 +- legendary/models/manifest.py | 6 +++--- 6 files changed, 12 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c5fd0b5..974619c 100644 --- a/README.md +++ b/README.md @@ -493,7 +493,7 @@ Legendary supports some options as well as game specific configuration in `~/.co log_level = debug ; maximum shared memory (in MiB) to use for installation max_memory = 1024 -; maximum number of worker processes when downloading (fewer wokers will be slower, but also use fewer system resources) +; maximum number of worker processes when downloading (fewer workers will be slower, but also use fewer system resources) max_workers = 8 ; default install directory install_dir = /mnt/tank/games @@ -511,7 +511,7 @@ wine_executable = wine ; wine prefix (alternative to using environment variable) wine_prefix = /home/user/.wine -; default environment variables to set (overriden by game specific ones) +; default environment variables to set (overridden by game specific ones) [default.env] WINEPREFIX = /home/user/legendary/.wine diff --git a/legendary/cli.py b/legendary/cli.py index de1345d..bf17aef 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -675,7 +675,7 @@ class LegendaryCLI: except Exception as e: end_t = time.time() logger.info(f'Installation failed after {end_t - start_t:.02f} seconds.') - logger.warning(f'The following exception occured while waiting for the donlowader to finish: {e!r}. ' + logger.warning(f'The following exception occurred while waiting for the downloader to finish: {e!r}. ' f'Try restarting the process, the resume file will be used to start where it failed. ' f'If it continues to fail please open an issue on GitHub.') else: @@ -890,7 +890,7 @@ class LegendaryCLI: f'with legendary. Run "legendary repair {args.app_name}" to do so.') else: logger.info(f'Installation had Epic Games Launcher metadata for version "{igame.version}", ' - f'verification will not be requried.') + f'verification will not be required.') logger.info('Game has been imported.') def egs_sync(self, args): @@ -1151,7 +1151,7 @@ def main(): install_parser.add_argument('--disable-patching', dest='disable_patching', action='store_true', help='Do not attempt to patch existing installation (download entire changed files)') install_parser.add_argument('--download-only', '--no-install', dest='no_install', action='store_true', - help='Do not intall app and do not run prerequisite installers after download') + help='Do not install app and do not run prerequisite installers after download') install_parser.add_argument('--update-only', dest='update_only', action='store_true', help='Only update, do not do anything if specified app is not installed') install_parser.add_argument('--dlm-debug', dest='dlm_debug', action='store_true', diff --git a/legendary/core.py b/legendary/core.py index 9b0225a..ab50656 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -187,7 +187,7 @@ class LegendaryCore: return True def get_assets(self, update_assets=False, platform_override=None) -> List[GameAsset]: - # do not save and always fetch list when platform is overriden + # do not save and always fetch list when platform is overridden if platform_override: return [GameAsset.from_egs_json(a) for a in self.egs.get_game_assets(platform=platform_override)] @@ -999,7 +999,7 @@ class LegendaryCore: if mf and os.path.exists(os.path.join(app_path, '.egstore', mf)): manifest_data = open(os.path.join(app_path, '.egstore', mf), 'rb').read() else: - self.log.warning('.egstore folder exists but manifest file is missing, contiuing as regular import...') + self.log.warning('.egstore folder exists but manifest file is missing, continuing as regular import...') # If there's no in-progress installation assume the game doesn't need to be verified if mf and not os.path.exists(os.path.join(app_path, '.egstore', 'bps')): @@ -1216,4 +1216,3 @@ class LegendaryCore: Do cleanup, config saving, and exit. """ self.lgd.save_config() - diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 6aa3fc6..0b84d2a 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -344,7 +344,7 @@ class DLManager(Process): self.tasks.append(FileTask(current_file.filename + u'.tmp', fopen=True)) self.tasks.extend(chunk_tasks) self.tasks.append(FileTask(current_file.filename + u'.tmp', close=True)) - # delete old file and rename temproary + # delete old file and rename temporary self.tasks.append(FileTask(current_file.filename, delete=True, rename=True, temporary_filename=current_file.filename + u'.tmp')) else: diff --git a/legendary/models/downloading.py b/legendary/models/downloading.py index 7907fd4..b538090 100644 --- a/legendary/models/downloading.py +++ b/legendary/models/downloading.py @@ -97,7 +97,7 @@ class SharedMemorySegment: class ChunkTask: def __init__(self, chunk_guid, chunk_offset=0, chunk_size=0, cleanup=False, chunk_file=None): """ - Download amanger chunk task + Download manager chunk task :param chunk_guid: GUID of chunk :param cleanup: whether or not this chunk can be removed from disk/memory after it has been written diff --git a/legendary/models/manifest.py b/legendary/models/manifest.py index 89b0d9c..8622f18 100644 --- a/legendary/models/manifest.py +++ b/legendary/models/manifest.py @@ -595,9 +595,9 @@ class FileManifest: return ''.format( - self.filename, self.symlink_target, self.hash.hex(), self.flags, - ', '.join(self.install_tags), cp_repr, self.file_size - ) + self.filename, self.symlink_target, self.hash.hex(), self.flags, + ', '.join(self.install_tags), cp_repr, self.file_size + ) class ChunkPart: From a2280edea8f7f8da9a080fd3fb2bafcabf9ee33d Mon Sep 17 00:00:00 2001 From: derrod Date: Sun, 10 Jan 2021 14:24:06 +0100 Subject: [PATCH 096/101] [cli/core] Add option to override launch executable --- legendary/cli.py | 5 ++++- legendary/core.py | 14 +++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/legendary/cli.py b/legendary/cli.py index bf17aef..9c3f613 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -474,7 +474,8 @@ class LegendaryCLI: extra_args=extra, user=args.user_name_override, wine_bin=args.wine_bin, wine_pfx=args.wine_pfx, language=args.language, wrapper=args.wrapper, - disable_wine=args.no_wine) + disable_wine=args.no_wine, + executable_override=args.executable_override) if args.set_defaults: self.core.lgd.config[app_name] = dict() @@ -1202,6 +1203,8 @@ def main(): help='Save parameters used to launch to config (does not include env vars)') launch_parser.add_argument('--reset-defaults', dest='reset_defaults', action='store_true', help='Reset config settings for app and exit') + launch_parser.add_argument('--override-exe', dest='executable_override', action='store', metavar='', + help='Override executable to launch (relative path)') if os.name != 'nt': launch_parser.add_argument('--wine', dest='wine_bin', action='store', metavar='', diff --git a/legendary/core.py b/legendary/core.py index ab50656..0c5e212 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -296,7 +296,8 @@ class LegendaryCore: user: str = None, extra_args: list = None, wine_bin: str = None, wine_pfx: str = None, language: str = None, wrapper: str = None, - disable_wine: bool = False) -> (list, str, dict): + disable_wine: bool = False, + executable_override: str = None) -> (list, str, dict): install = self.lgd.get_installed_game(app_name) game = self.lgd.get_game_meta(app_name) @@ -312,8 +313,15 @@ class LegendaryCore: if user: user_name = user - game_exe = os.path.join(install.install_path, - install.executable.replace('\\', '/').lstrip('/')) + if executable_override or (executable_override := self.lgd.config.get(app_name, 'override_exe', fallback=None)): + game_exe = os.path.join(install.install_path, + executable_override.replace('\\', '/')) + if not os.path.exists(game_exe): + raise ValueError(f'Executable path is invalid: {game_exe}') + else: + game_exe = os.path.join(install.install_path, + install.executable.replace('\\', '/').lstrip('/')) + working_dir = os.path.split(game_exe)[0] params = [] From edad963200d27d0078ea8bbf9993c0942508f6e9 Mon Sep 17 00:00:00 2001 From: derrod Date: Sun, 10 Jan 2021 14:25:35 +0100 Subject: [PATCH 097/101] [models/utils] Track file offset for chunk parts Preparation for smarter patching with downloader rewrite. --- legendary/models/json_manifest.py | 3 +++ legendary/models/manifest.py | 14 +++++++++++--- legendary/utils/savegame_helper.py | 3 ++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/legendary/models/json_manifest.py b/legendary/models/json_manifest.py index 54ad2e3..e43ad75 100644 --- a/legendary/models/json_manifest.py +++ b/legendary/models/json_manifest.py @@ -157,15 +157,18 @@ class JSONFML(FML): _fm.chunk_parts = [] _fm.install_tags = _fmj.pop('InstallTags', list()) + _offset = 0 for _cpj in _fmj.pop('FileChunkParts'): _cp = ChunkPart() _cp.guid = guid_from_json(_cpj.pop('Guid')) _cp.offset = blob_to_num(_cpj.pop('Offset')) _cp.size = blob_to_num(_cpj.pop('Size')) + _cp.file_offset = _offset _fm.file_size += _cp.size if _cpj: print(f'Non-read ChunkPart keys: {_cpj.keys()}') _fm.chunk_parts.append(_cp) + _offset += _cp.size if _fmj: print(f'Non-read FileManifest keys: {_fmj.keys()}') diff --git a/legendary/models/manifest.py b/legendary/models/manifest.py index 8622f18..c66b9cb 100644 --- a/legendary/models/manifest.py +++ b/legendary/models/manifest.py @@ -507,13 +507,20 @@ class FML: # Each file is made up of "Chunk Parts" that can be spread across the "chunk stream" for fm in _fml.elements: _elem = struct.unpack(' 0: + logger.warning(f'Did not read {diff} bytes from chunk part!') + bio.seek(diff) # we have to calculate the actual file size ourselves for fm in _fml.elements: @@ -601,10 +608,11 @@ class FileManifest: class ChunkPart: - def __init__(self, guid=None, offset=0, size=0): + def __init__(self, guid=None, offset=0, size=0, file_offset=0): self.guid = guid self.offset = offset self.size = size + self.file_offset = file_offset # caches for things that are "expensive" to compute self._guid_str = None self._guid_num = None @@ -623,8 +631,8 @@ class ChunkPart: def __repr__(self): guid_readable = '-'.join('{:08x}'.format(g) for g in self.guid) - return ''.format( - guid_readable, self.offset, self.size) + return ''.format( + guid_readable, self.offset, self.size, self.file_offset) class CustomFields: diff --git a/legendary/utils/savegame_helper.py b/legendary/utils/savegame_helper.py index 48f4a8f..d7b36e8 100644 --- a/legendary/utils/savegame_helper.py +++ b/legendary/utils/savegame_helper.py @@ -123,7 +123,8 @@ class SaveGameHelper: # create chunk part and write it to chunk buffer cp = ChunkPart(guid=cur_chunk.guid, offset=cur_buffer.tell(), - size=min(remaining, 1024 * 1024 - cur_buffer.tell())) + size=min(remaining, 1024 * 1024 - cur_buffer.tell()), + file_offset=cf.tell()) _tmp = cf.read(cp.size) if not _tmp: self.log.warning(f'Got EOF for "{f.filename}" with {remaining} bytes remaining! ' From f040f4bd4040c71e6c1f355c8baf7d2eaa5bb607 Mon Sep 17 00:00:00 2001 From: derrod Date: Wed, 20 Jan 2021 14:43:32 +0100 Subject: [PATCH 098/101] [api] EGS: Add method to fetch library items This is required for us to also get items that don't have assets (e.g. some DLCs/Origin-managed games) --- legendary/api/egs.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/legendary/api/egs.py b/legendary/api/egs.py index cd73817..01c850a 100644 --- a/legendary/api/egs.py +++ b/legendary/api/egs.py @@ -21,6 +21,7 @@ class EPCAPI: _catalog_host = 'catalog-public-service-prod06.ol.epicgames.com' _ecommerce_host = 'ecommerceintegration-public-service-ecomprod02.ol.epicgames.com' _datastorage_host = 'datastorage-public-service-liveegs.live.use1a.on.epicgames.com' + _library_host = 'library-service.live.use1a.on.epicgames.com' def __init__(self, lc='en', cc='US'): self.session = requests.session() @@ -122,6 +123,24 @@ class EPCAPI: r.raise_for_status() return r.json().get(catalog_item_id, None) + def get_library_items(self, include_metadata=True): + records = [] + r = self.session.get(f'https://{self._library_host}/library/api/public/items', + params=dict(includeMetadata=include_metadata)) + r.raise_for_status() + j = r.json() + records.extend(j['records']) + + # Fetch remaining library entries as long as there is a cursor + while cursor := j['responseMetadata'].get('nextCursor', None): + r = self.session.get(f'https://{self._library_host}/library/api/public/items', + params=dict(includeMetadata=include_metadata, cursor=cursor)) + r.raise_for_status() + j = r.json() + records.extend(j['records']) + + return records + def get_user_cloud_saves(self, app_name='', manifests=False, filenames=None): if app_name and manifests: app_name += '/manifests/' From 66fae77ff91aede3e628f924c5fd935308c47412 Mon Sep 17 00:00:00 2001 From: derrod Date: Fri, 22 Jan 2021 15:24:16 +0100 Subject: [PATCH 099/101] .github: Add issue template --- .github/ISSUE_TEMPLATE/bug_report.md | 34 ++++++++++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 8 +++++++ 2 files changed, 42 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..f1cf654 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,34 @@ + + + + + + + + + + +## Platform + + +Operating system and version: +Legendary version (`legendary -V`): + +## Expected Behavior + + +## Current Behavior + + + +## Steps to Reproduce + + +1. +2. +3. +4. + +## Additional information + + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..3b055ae --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: GitHub Discussions + url: https://github.com/derrod/legendary/discussions + about: GitHub Forum for anything that is not a legendary issue (e.g. game or WINE problems) + - name: Discord chat + url: https://discord.gg/RQHbMVrwRr + about: Discord chat for help with game or WINE issues From 081bac7ef21e5d1c2aff0ad8fcd3f06cc65360fb Mon Sep 17 00:00:00 2001 From: derrod Date: Sat, 30 Jan 2021 09:34:28 +0100 Subject: [PATCH 100/101] .github: Fix bug report and add feature request template --- .github/ISSUE_TEMPLATE/bug_report.md | 6 ++++++ .github/ISSUE_TEMPLATE/feature_request.md | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index f1cf654..967a413 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,3 +1,9 @@ +--- +name: Legendary bug report +about: Legendary crashes or bugs (not WINE/Game crashes!) +labels: '' +assignees: '' +--- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..3dfb90e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,18 @@ +--- +name: Feature request +about: Request features that are missing (compared to EGS) or new ones for improving Legendary itself. +labels: '' +assignees: '' +--- + +**Is your feature request related to a problem? Please describe.** + + +**Describe the solution you'd like** + + +**Describe alternatives you've considered** + + +**Additional context** + From 313323e43a05eed57574977367fa7300bed26560 Mon Sep 17 00:00:00 2001 From: derrod Date: Sun, 31 Jan 2021 21:26:07 +0100 Subject: [PATCH 101/101] .github: Add link to Wiki to issue config --- .github/ISSUE_TEMPLATE/config.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 3b055ae..111c719 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,8 +1,11 @@ -blank_issues_enabled: false -contact_links: - - name: GitHub Discussions - url: https://github.com/derrod/legendary/discussions - about: GitHub Forum for anything that is not a legendary issue (e.g. game or WINE problems) - - name: Discord chat - url: https://discord.gg/RQHbMVrwRr - about: Discord chat for help with game or WINE issues +blank_issues_enabled: false +contact_links: + - name: GitHub Wiki + url: https://github.com/derrod/legendary/wiki/Game-workarounds + about: The Legendary Wiki contains troubleshooting steps for some games and a guide for setting up Proton + - name: GitHub Discussions + url: https://github.com/derrod/legendary/discussions + about: GitHub Forum for anything that is not a legendary issue (e.g. game or WINE problems) + - name: Discord chat + url: https://discord.gg/RQHbMVrwRr + about: Discord chat for help with game or WINE issues