diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..967a413 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,40 @@ +--- +name: Legendary bug report +about: Legendary crashes or bugs (not WINE/Game crashes!) +labels: '' +assignees: '' +--- + + + + + + + + + + +## Platform + + +Operating system and version: +Legendary version (`legendary -V`): + +## Expected Behavior + + +## Current Behavior + + + +## Steps to Reproduce + + +1. +2. +3. +4. + +## Additional information + + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..111c719 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + - name: GitHub Wiki + url: https://github.com/derrod/legendary/wiki/Game-workarounds + about: The Legendary Wiki contains troubleshooting steps for some games and a guide for setting up Proton + - name: GitHub Discussions + url: https://github.com/derrod/legendary/discussions + about: GitHub Forum for anything that is not a legendary issue (e.g. game or WINE problems) + - name: Discord chat + url: https://discord.gg/RQHbMVrwRr + about: Discord chat for help with game or WINE issues diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..3dfb90e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,18 @@ +--- +name: Feature request +about: Request features that are missing (compared to EGS) or new ones for improving Legendary itself. +labels: '' +assignees: '' +--- + +**Is your feature request related to a problem? Please describe.** + + +**Describe the solution you'd like** + + +**Describe alternatives you've considered** + + +**Additional context** + diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a7f94b7..93b82a9 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -47,3 +47,38 @@ jobs: with: name: ${{ runner.os }}-package path: legendary/dist/* + + deb: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ['ubuntu-20.04'] + + fail-fast: true + max-parallel: 3 + + steps: + - uses: actions/checkout@v2 + + - name: Dependencies + run: sudo apt install + python3-all + python3-stdeb + dh-python + python3-requests + python3-setuptools + python3-wheel + + - name: Build + run: python3 setup.py --command-packages=stdeb.command bdist_deb + + - name: Os version + id: os_version + run: | + source /etc/os-release + echo ::set-output name=version::$NAME-$VERSION_ID + + - uses: actions/upload-artifact@v2 + with: + name: ${{ steps.os_version.outputs.version }}-deb-package + path: deb_dist/*.deb diff --git a/README.md b/README.md index 4a762d9..974619c 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,14 @@ [![Discord](https://discordapp.com/api/guilds/695233346627698689/widget.png?style=shield)](https://discord.gg/UJKBwPw) [![Twitter Follow](https://img.shields.io/twitter/follow/legendary_gl?label=Follow%20us%20for%20updates%21&style=social)](https://twitter.com/legendary_gl) Legendary is an open-source game launcher that can download and install games from the Epic Games platform on Linux and Windows. -It's name as a tongue-in-cheek play on tiers of [item rarity in many MMORPGs](https://wow.gamepedia.com/Quality). +Its name as a tongue-in-cheek play on tiers of [item rarity in many MMORPGs](https://wow.gamepedia.com/Quality). Right now Legendary is in beta and not feature-complete. You might run into some bugs or issues. If you do please [create an issue on GitHub](https://github.com/derrod/legendary/issues/new) so we can fix it. +**Note:** Legendary is currently a CLI (command-line interface) application without a graphical user interface, +it has to be run from a terminal (e.g. PowerShell) + **What works:** - Authenticating with Epic's service - Downloading and installing your games and their DLC @@ -47,6 +50,7 @@ Note that since packages are maintained by third parties it may take a bit for t If you always want to have the latest features and fixes available then using the PyPI distribution is recommended. ### Standalone + Download the `legendary` or `legendary.exe` binary from [the latest release](https://github.com/derrod/legendary/releases/latest) and move it to somewhere in your `$PATH`/`%PATH%`. Don't forget to `chmod +x` it on Linux. @@ -55,15 +59,25 @@ Note that on Linux glibc >= 2.25 is required, so older distributions such as Ubu ### Python package +#### Prerequisites + +To prevent problems with permissions during installation, please upgrade your `pip` by running `python -m pip install -U pip --user`. + +> **Tip:** You may need to replace `python` in the above command with `python3.8` on Linux, or `py -3.8` on Windows. + +#### Installation from PyPI (recommended) + Legendary is available on [PyPI](https://pypi.org/project/legendary-gl/), to install simply run: + ```bash pip install legendary-gl ``` #### Manually from the repo + - Install python3.8, setuptools, wheel, and requests - Clone the git repository and cd into it -- Run `python3.8 setup.py install` +- Run `pip install .` #### Ubuntu 20.04 example @@ -72,81 +86,91 @@ Ubuntu 20.04's standard repositories include everything needed to install legend sudo apt install python3 python3-requests python3-setuptools-git git clone https://github.com/derrod/legendary.git cd legendary -sudo python3 setup.py install +pip install . ```` -Note that in this example we used `sudo` to install the package on the system, this may not be advisable depending on your setup. +If the `legendary` executable is not available after installation, you may need to configure your `PATH` correctly. You can do this by running the command: + +```bash +echo 'export PATH=$PATH:~/.local/bin' >> ~/.profile && source ~/.profile +``` ### Directly from the repo (for dev/testing) - Install python3.8 and requests (optionally in a venv) -- cd into `legendary/` (the folder with `cli.py`) -- run `PYTHONPATH=.. python3.8 cli.py` +- cd into the repository +- Run `pip install -e .` + +This installs `legendary` in "editable" mode - any changes to the source code will take effect next time the `legendary` executable runs. ## Quickstart +**Tip:** When using PowerShell with the standalone executable, you may need to replace `legendary` with `.\legendary` in the commands below. + To log in: ```` -$ legendary auth +legendary auth ```` Authentication is a little finicky since we have to go through the Epic website. The login page should open in your browser and after logging in you should be presented with a JSON response that contains a code, just copy and paste the code into your terminal to log in. On Windows you can use the `--import` flag to import the authentication from the Epic Games Launcher. Note that this will log you out of the Epic Launcher. Listing your games ```` -$ legendary list-games +legendary list-games ```` This will fetch a list of games available on your account, the first time may take a while depending on how many games you have. Installing a game ```` -$ legendary install Anemone +legendary install Anemone ```` **Important:** the name used for these commands is the app name, *not* the game's name! The app name is in the parentheses after the game title in the games list. List installed games and check for updates ```` -$ legendary list-installed --check-updates +legendary list-installed --check-updates ```` Launch (run) a game with online authentication ```` -$ legendary launch Anemone +legendary launch Anemone ```` **Tip:** most games will run fine offline (`--offline`), and thus won't require launching through legendary for online authentication. You can run `legendary launch --offline --dry-run` to get a command line that will launch the game with all parameters that would be used by the Epic Launcher. These can then be entered into any other game launcher (e.g. Lutris/Steam) if the game requires them. Importing a previously installed game ```` -$ legendary import-game Anemone /mnt/games/Epic/WorldOfGoo +legendary import-game Anemone /mnt/games/Epic/WorldOfGoo ```` **Note:** Importing will require a full verification so Legendary can correctly update the game later. Sync savegames with the Epic Cloud ```` -$ legendary sync-saves +legendary sync-saves ```` **Note:** When this command is run the first time after a supported game has been installed it will ask you to confirm or provide the path to where the savegame is located. Automatically sync all games with the Epic Games Launcher ```` -$ legendary -y egl-sync +legendary -y egl-sync ```` ## Usage ```` -usage: legendary [-h] [-v] [-y] [-V] {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync} ... +usage: legendary [-h] [-v] [-y] [-V] + {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync,status,cleanup} + ... -Legendary v0.0.X - "Codename" +Legendary v0.X.X - "Codename" optional arguments: -h, --help show this help message and exit - -v Set loglevel to debug + -v, --debug Set loglevel to debug -y, --yes Default to yes for all prompts - -V Print version and exit + -V, --version Print version and exit Commands: - {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync} + {auth,install,download,update,repair,uninstall,launch,list-games,list-installed,list-files,list-saves,download-saves,sync-saves,verify-game,import-game,egl-sync,status,cleanup} auth Authenticate with EPIC install (download,update,repair) Download a game @@ -161,18 +185,24 @@ Commands: verify-game Verify a game's local files import-game Import an already installed game egl-sync Setup or run Epic Games Launcher sync + status Show legendary status information + cleanup Remove old temporary, metadata, and manifest files Individual command help: Command: auth -usage: legendary auth [-h] [--import] [--code ] [--sid ] [--delete] +usage: legendary auth [-h] [--import] [--code ] + [--sid ] [--delete] optional arguments: -h, --help show this help message and exit - --import Import Epic Games Launcher authentication data (logs out of EGL) + --import Import Epic Games Launcher authentication data (logs + out of EGL) --code - Use specified exchange code instead of interactive authentication - --sid Use specified session id instead of interactive authentication + Use specified exchange code instead of interactive + authentication + --sid Use specified session id instead of interactive + authentication --delete Remove existing authentication (log out) @@ -187,38 +217,66 @@ positional arguments: optional arguments: -h, --help show this help message and exit --base-path Path for game installations (defaults to ~/legendary) - --game-folder Folder for game installation (defaults to folder specified in metadata) + --game-folder Folder for game installation (defaults to folder + specified in metadata) --max-shared-memory - Maximum amount of shared memory to use (in MiB), default: 1 GiB - --max-workers Maximum amount of download workers, default: min(2 * CPUs, 16) - --manifest Manifest URL or path to use instead of the CDN one (e.g. for downgrading) - --old-manifest Manifest URL or path to use as the old one (e.g. for testing patching) - --base-url Base URL to download from (e.g. to test or switch to a different CDNs) + Maximum amount of shared memory to use (in MiB), + default: 1 GiB + --max-workers Maximum amount of download workers, default: min(2 * + CPUs, 16) + --manifest Manifest URL or path to use instead of the CDN one + (e.g. for downgrading) + --old-manifest Manifest URL or path to use as the old one (e.g. for + testing patching) + --delta-manifest + Manifest URL or path to use as the delta one (e.g. for + testing) + --base-url Base URL to download from (e.g. to test or switch to a + different CDNs) --force Download all files / ignore existing (overwrite) - --disable-patching Do not attempt to patch existing installation (download entire changed files) + --disable-patching Do not attempt to patch existing installation + (download entire changed files) --download-only, --no-install - Do not intall app and do not run prerequisite installers after download - --update-only Only update, do not do anything if specified app is not installed - --dlm-debug Set download manager and worker processes' loglevel to debug + Do not intall app and do not run prerequisite + installers after download + --update-only Only update, do not do anything if specified app is + not installed + --dlm-debug Set download manager and worker processes' loglevel to + debug --platform - Platform override for download (also sets --no-install) - --prefix Only fetch files whose path starts with (case insensitive) - --exclude Exclude files starting with (case insensitive) + Platform override for download (also sets --no- + install) + --prefix Only fetch files whose path starts with (case + insensitive) + --exclude Exclude files starting with (case + insensitive) --install-tag Only download files with the specified install tag - --enable-reordering Enable reordering optimization to reduce RAM requirements during download (may have adverse results for some titles) - --dl-timeout Connection timeout for downloader (default: 10 seconds) + --enable-reordering Enable reordering optimization to reduce RAM + requirements during download (may have adverse results + for some titles) + --dl-timeout Connection timeout for downloader (default: 10 + seconds) --save-path Set save game path to be used for sync-saves - --repair Repair installed game by checking and redownloading corrupted/missing files + --repair Repair installed game by checking and redownloading + corrupted/missing files + --repair-and-update Update game to the latest version when repairing + --ignore-free-space Do not abort if not enough free space is available + --disable-delta-manifests + Do not use delta manifests when updating (may increase + download size) + --reset-sdl Reset selective downloading choices (requires repair + to download new components) Command: uninstall -usage: legendary uninstall [-h] +usage: legendary uninstall [-h] [--keep-files] positional arguments: - Name of the app + Name of the app optional arguments: - -h, --help show this help message and exit + -h, --help show this help message and exit + --keep-files Keep files but remove game from Legendary database Command: launch @@ -231,16 +289,21 @@ positional arguments: optional arguments: -h, --help show this help message and exit - --offline Skip login and launch game without online authentication + --offline Skip login and launch game without online + authentication --skip-version-check Skip version check when launching game in online mode --override-username - Override username used when launching the game (only works with some titles) - --dry-run Print the command line that would have been used to launch the game and exit + Override username used when launching the game (only + works with some titles) + --dry-run Print the command line that would have been used to + launch the game and exit --language - Override language for game launch (defaults to system locale) + Override language for game launch (defaults to system + locale) --wrapper Wrapper command to launch game with - --set-defaults Save parameters used to launch to config (does not include env vars) + --set-defaults Save parameters used to launch to config (does not + include env vars) --reset-defaults Reset config settings for app and exit --wine Set WINE binary to use to launch the app --wine-prefix @@ -249,30 +312,39 @@ optional arguments: Command: list-games -usage: legendary list-games [-h] [--platform ] [--include-ue] [--csv] [--tsv] +usage: legendary list-games [-h] [--platform ] [--include-ue] [--csv] + [--tsv] [--json] optional arguments: -h, --help show this help message and exit --platform - Override platform that games are shown for (e.g. Win32/Mac) - --include-ue Also include Unreal Engine content (Engine/Marketplace) in list + Override platform that games are shown for (e.g. + Win32/Mac) + --include-ue Also include Unreal Engine content + (Engine/Marketplace) in list --csv List games in CSV format --tsv List games in TSV format + --json List games in JSON format Command: list-installed -usage: legendary list-installed [-h] [--check-updates] [--csv] [--tsv] [--show-dirs] +usage: legendary list-installed [-h] [--check-updates] [--csv] [--tsv] [--json] + [--show-dirs] optional arguments: -h, --help show this help message and exit --check-updates Check for updates for installed games --csv List games in CSV format --tsv List games in TSV format + --json List games in JSON format --show-dirs Print installation directory in output Command: list-files -usage: legendary list-files [-h] [--force-download] [--platform ] [--manifest ] [--csv] [--tsv] [--hashlist] [--install-tag ] [] +usage: legendary list-files [-h] [--force-download] [--platform ] + [--manifest ] [--csv] [--tsv] [--json] + [--hashlist] [--install-tag ] + [] positional arguments: Name of the app (optional) @@ -285,7 +357,9 @@ optional arguments: --manifest Manifest URL or path to use instead of the CDN one --csv Output in CSV format --tsv Output in TSV format - --hashlist Output file hash list in hashcheck/sha1sum -c compatible format + --json Output in JSON format + --hashlist Output file hash list in hashcheck/sha1sum -c + compatible format --install-tag Show only files with specified install tag @@ -310,7 +384,10 @@ optional arguments: Command: sync-saves -usage: legendary sync-saves [-h] [--skip-upload] [--skip-download] [--force-upload] [--force-download] [--save-path ] [--disable-filters] [] +usage: legendary sync-saves [-h] [--skip-upload] [--skip-download] + [--force-upload] [--force-download] + [--save-path ] [--disable-filters] + [] positional arguments: Name of the app (optional) @@ -321,7 +398,8 @@ optional arguments: --skip-download Only upload new saves from cloud, don't download --force-upload Force upload even if local saves are older --force-download Force download even if local saves are newer - --save-path Override savegame path (requires single app name to be specified) + --save-path Override savegame path (requires single app name to be + specified) --disable-filters Disable save game file filtering @@ -336,7 +414,8 @@ optional arguments: Command: import-game -usage: legendary import-game [-h] [--disable-check] +usage: legendary import-game [-h] [--disable-check] + positional arguments: Name of the app @@ -345,24 +424,50 @@ positional arguments: optional arguments: -h, --help show this help message and exit - --disable-check Disables completeness check of the to-be-imported game installation (useful if the imported game is a much older version or missing files) + --disable-check Disables completeness check of the to-be-imported game + installation (useful if the imported game is a much + older version or missing files) Command: egl-sync -usage: legendary egl-sync [-h] [--egl-manifest-path EGL_MANIFEST_PATH] [--egl-wine-prefix EGL_WINE_PREFIX] [--enable-sync] [--disable-sync] [--one-shot] [--import-only] [--export-only] [--unlink] +usage: legendary egl-sync [-h] [--egl-manifest-path EGL_MANIFEST_PATH] + [--egl-wine-prefix EGL_WINE_PREFIX] [--enable-sync] + [--disable-sync] [--one-shot] [--import-only] + [--export-only] [--unlink] optional arguments: -h, --help show this help message and exit --egl-manifest-path EGL_MANIFEST_PATH - Path to the Epic Games Launcher's Manifests folder, should point to /ProgramData/Epic/EpicGamesLauncher/Data/Manifests + Path to the Epic Games Launcher's Manifests folder, + should point to + /ProgramData/Epic/EpicGamesLauncher/Data/Manifests --egl-wine-prefix EGL_WINE_PREFIX - Path to the WINE prefix the Epic Games Launcher is installed in + Path to the WINE prefix the Epic Games Launcher is + installed in --enable-sync Enable automatic EGL <-> Legendary sync --disable-sync Disable automatic sync and exit --one-shot Sync once, do not ask to setup automatic sync --import-only Only import games from EGL (no export) --export-only Only export games to EGL (no import) - --unlink Disable sync and remove EGL metadata from installed games + --unlink Disable sync and remove EGL metadata from installed + games + + +Command: status +usage: legendary status [-h] [--offline] [--json] + +optional arguments: + -h, --help show this help message and exit + --offline Only print offline status information, do not login + --json Show status in JSON format + + +Command: cleanup +usage: legendary cleanup [-h] [--keep-manifests] + +optional arguments: + -h, --help show this help message and exit + --keep-manifests Do not delete old manifests ```` @@ -388,6 +493,8 @@ Legendary supports some options as well as game specific configuration in `~/.co log_level = debug ; maximum shared memory (in MiB) to use for installation max_memory = 1024 +; maximum number of worker processes when downloading (fewer workers will be slower, but also use fewer system resources) +max_workers = 8 ; default install directory install_dir = /mnt/tank/games ; locale override, must be in RFC 1766 format (e.g. "en-US") @@ -404,7 +511,7 @@ wine_executable = wine ; wine prefix (alternative to using environment variable) wine_prefix = /home/user/.wine -; default environment variables to set (overriden by game specific ones) +; default environment variables to set (overridden by game specific ones) [default.env] WINEPREFIX = /home/user/legendary/.wine @@ -427,7 +534,8 @@ DXVK_CONFIG_FILE = /mnt/tank/games/Game/dxvk.conf [AppName2] ; Use a wrapper to run this script -wrapper = /path/to/wrapper --parameters +; Note that the path might have to be quoted if it contains spaces +wrapper = "/path/to/Proton 5.0/proton" run ; Do not run this executable with WINE (e.g. when the wrapper handles that) no_wine = true ```` diff --git a/legendary/__init__.py b/legendary/__init__.py index aeaa166..1760b2a 100644 --- a/legendary/__init__.py +++ b/legendary/__init__.py @@ -1,4 +1,4 @@ """Legendary!""" -__version__ = '0.0.19' -__codename__ = 'Interloper' +__version__ = '0.20.6' +__codename__ = 'A Red Letter Day' diff --git a/legendary/api/egs.py b/legendary/api/egs.py index 992b73c..01c850a 100644 --- a/legendary/api/egs.py +++ b/legendary/api/egs.py @@ -10,7 +10,7 @@ from legendary.models.exceptions import InvalidCredentialsError class EPCAPI: - _user_agent = 'UELauncher/10.16.1-13343695+++Portal+Release-Live Windows/10.0.18363.1.256.64bit' + _user_agent = 'UELauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit' # required for the oauth request _user_basic = '34a02cf8f4414e29b15921876da36f9a' _pw_basic = 'daafbccc737745039dffe53d94fc76cf' @@ -21,6 +21,7 @@ class EPCAPI: _catalog_host = 'catalog-public-service-prod06.ol.epicgames.com' _ecommerce_host = 'ecommerceintegration-public-service-ecomprod02.ol.epicgames.com' _datastorage_host = 'datastorage-public-service-liveegs.live.use1a.on.epicgames.com' + _library_host = 'library-service.live.use1a.on.epicgames.com' def __init__(self, lc='en', cc='US'): self.session = requests.session() @@ -122,6 +123,24 @@ class EPCAPI: r.raise_for_status() return r.json().get(catalog_item_id, None) + def get_library_items(self, include_metadata=True): + records = [] + r = self.session.get(f'https://{self._library_host}/library/api/public/items', + params=dict(includeMetadata=include_metadata)) + r.raise_for_status() + j = r.json() + records.extend(j['records']) + + # Fetch remaining library entries as long as there is a cursor + while cursor := j['responseMetadata'].get('nextCursor', None): + r = self.session.get(f'https://{self._library_host}/library/api/public/items', + params=dict(includeMetadata=include_metadata, cursor=cursor)) + r.raise_for_status() + j = r.json() + records.extend(j['records']) + + return records + def get_user_cloud_saves(self, app_name='', manifests=False, filenames=None): if app_name and manifests: app_name += '/manifests/' diff --git a/legendary/cli.py b/legendary/cli.py index f1c61e9..7441fd1 100644 --- a/legendary/cli.py +++ b/legendary/cli.py @@ -3,6 +3,7 @@ import argparse import csv +import json import logging import os import shlex @@ -20,9 +21,10 @@ from legendary import __version__, __codename__ from legendary.core import LegendaryCore from legendary.models.exceptions import InvalidCredentialsError from legendary.models.game import SaveGameStatus, VerifyResult -from legendary.utils.cli import get_boolean_choice +from legendary.utils.cli import get_boolean_choice, sdl_prompt from legendary.utils.custom_parser import AliasedSubParsersAction from legendary.utils.lfs import validate_files +from legendary.utils.selective_dl import get_sdl_appname # todo custom formatter for cli logger (clean info, highlighted error/warning) logging.basicConfig( @@ -146,9 +148,9 @@ class LegendaryCLI: platform_override=args.platform_override, skip_ue=not args.include_ue ) # sort games and dlc by name - games = sorted(games, key=lambda x: x.app_title) + games = sorted(games, key=lambda x: x.app_title.lower()) for citem_id in dlc_list.keys(): - dlc_list[citem_id] = sorted(dlc_list[citem_id], key=lambda d: d.app_title) + dlc_list[citem_id] = sorted(dlc_list[citem_id], key=lambda d: d.app_title.lower()) if args.csv or args.tsv: writer = csv.writer(stdout, dialect='excel-tab' if args.tsv else 'excel') @@ -159,6 +161,16 @@ class LegendaryCLI: writer.writerow((dlc.app_name, dlc.app_title, dlc.app_version, True)) return + if args.json: + _out = [] + for game in games: + _j = vars(game) + _j['dlcs'] = [vars(dlc) for dlc in dlc_list[game.asset_info.catalog_item_id]] + _out.append(_j) + + print(json.dumps(_out, sort_keys=True, indent=2)) + return + print('\nAvailable games:') for game in games: print(f' * {game.app_title} (App name: {game.app_name} | Version: {game.app_version})') @@ -176,17 +188,28 @@ class LegendaryCLI: self.core.get_assets(True) games = sorted(self.core.get_installed_list(), - key=lambda x: x.title) + key=lambda x: x.title.lower()) versions = dict() for game in games: - versions[game.app_name] = self.core.get_asset(game.app_name).build_version + try: + versions[game.app_name] = self.core.get_asset(game.app_name).build_version + except ValueError: + logger.warning(f'Metadata for "{game.app_name}" is missing, the game may have been removed from ' + f'your account or not be in legendary\'s database yet, try rerunning the command ' + f'with "--check-updates".') if args.csv or args.tsv: writer = csv.writer(stdout, dialect='excel-tab' if args.tsv else 'excel') - writer.writerow(['App name', 'App title', 'Installed version', 'Available version', 'Update available']) + writer.writerow(['App name', 'App title', 'Installed version', 'Available version', + 'Update available', 'Install size', 'Install path']) writer.writerows((game.app_name, game.title, game.version, versions[game.app_name], - versions[game.app_name] != game.version) for game in games) + versions[game.app_name] != game.version, game.install_size, game.install_path) + for game in games if game.app_name in versions) + return + + if args.json: + print(json.dumps([vars(g) for g in games], indent=2, sort_keys=True)) return print('\nInstalled games:') @@ -203,7 +226,7 @@ class LegendaryCLI: print(f' + Location: {game.install_path}') if not os.path.exists(game.install_path): print(f' ! Game does no longer appear to be installed (directory "{game.install_path}" missing)!') - elif versions[game.app_name] != game.version: + elif game.app_name in versions and versions[game.app_name] != game.version: print(f' -> Update available! Installed: {game.version}, Latest: {versions[game.app_name]}') print(f'\nTotal: {len(games)}') @@ -229,6 +252,9 @@ class LegendaryCLI: logger.error('Login failed! Cannot continue with download process.') exit(1) game = self.core.get_game(args.app_name, update_meta=True) + if not game: + logger.fatal(f'Could not fetch metadata for "{args.app_name}" (check spelling/account ownership)') + exit(1) manifest_data, _ = self.core.get_cdn_manifest(game, platform_override=args.platform_override) manifest = self.core.load_manifest(manifest_data) @@ -244,7 +270,18 @@ class LegendaryCLI: elif args.csv or args.tsv: writer = csv.writer(stdout, dialect='excel-tab' if args.tsv else 'excel') writer.writerow(['path', 'hash', 'size', 'install_tags']) - writer.writerows((fm.filename, fm.hash.hex(), fm.file_size, '|'.join(fm.install_tags))for fm in files) + writer.writerows((fm.filename, fm.hash.hex(), fm.file_size, '|'.join(fm.install_tags)) for fm in files) + elif args.json: + _files = [] + for fm in files: + _files.append(dict( + filename=fm.filename, + sha_hash=fm.hash.hex(), + install_tags=fm.install_tags, + file_size=fm.file_size, + flags=fm.flags, + )) + print(json.dumps(_files, sort_keys=True, indent=2)) else: install_tags = set() for fm in files: @@ -424,7 +461,12 @@ class LegendaryCLI: if not args.skip_version_check and not self.core.is_noupdate_game(app_name): logger.info('Checking for updates...') - latest = self.core.get_asset(app_name, update=True) + try: + latest = self.core.get_asset(app_name, update=True) + except ValueError: + logger.fatal(f'Metadata for "{app_name}" does not exist, cannot launch!') + exit(1) + if latest.build_version != igame.version: logger.error('Game is out of date, please update or launch with update check skipping!') exit(1) @@ -433,7 +475,8 @@ class LegendaryCLI: extra_args=extra, user=args.user_name_override, wine_bin=args.wine_bin, wine_pfx=args.wine_pfx, language=args.language, wrapper=args.wrapper, - disable_wine=args.no_wine) + disable_wine=args.no_wine, + executable_override=args.executable_override) if args.set_defaults: self.core.lgd.config[app_name] = dict() @@ -483,7 +526,7 @@ class LegendaryCLI: args.no_install = True elif args.subparser_name == 'repair' or args.repair_mode: args.repair_mode = True - args.no_install = True + args.no_install = args.repair_and_update is False repair_file = os.path.join(self.core.lgd.get_tmp_path(), f'{args.app_name}.repair') if not self.core.login(): @@ -537,6 +580,17 @@ class LegendaryCLI: else: logger.info(f'Using existing repair file: {repair_file}') + # Workaround for Cyberpunk 2077 preload + if not args.install_tag and not game.is_dlc and ((sdl_name := get_sdl_appname(game.app_name)) is not None): + config_tags = self.core.lgd.config.get(game.app_name, 'install_tags', fallback=None) + if not self.core.is_installed(game.app_name) or config_tags is None or args.reset_sdl: + args.install_tag = sdl_prompt(sdl_name, game.app_title) + if game.app_name not in self.core.lgd.config: + self.core.lgd.config[game.app_name] = dict() + self.core.lgd.config.set(game.app_name, 'install_tags', ','.join(args.install_tag)) + else: + args.install_tag = config_tags.split(',') + logger.info('Preparing download...') # todo use status queue to print progress from CLI # This has become a little ridiculous hasn't it? @@ -553,19 +607,30 @@ class LegendaryCLI: file_install_tag=args.install_tag, dl_optimizations=args.order_opt, dl_timeout=args.dl_timeout, - repair=args.repair_mode) + repair=args.repair_mode, + repair_use_latest=args.repair_and_update, + disable_delta=args.disable_delta, + override_delta_manifest=args.override_delta_manifest) # game is either up to date or hasn't changed, so we have nothing to do if not analysis.dl_size: + old_igame = self.core.get_installed_game(game.app_name) logger.info('Download size is 0, the game is either already up to date or has not changed. Exiting...') - if args.repair_mode and os.path.exists(repair_file): - igame = self.core.get_installed_game(game.app_name) - if igame.needs_verification: - igame.needs_verification = False - self.core.install_game(igame) + if old_igame and args.repair_mode and os.path.exists(repair_file): + if old_igame.needs_verification: + old_igame.needs_verification = False + self.core.install_game(old_igame) logger.debug('Removing repair file.') os.remove(repair_file) + + # check if install tags have changed, if they did; try deleting files that are no longer required. + if old_igame and old_igame.install_tags != igame.install_tags: + old_igame.install_tags = igame.install_tags + self.logger.info('Deleting now untagged files.') + self.core.uninstall_tag(old_igame) + self.core.install_game(old_igame) + exit(0) logger.info(f'Install size: {analysis.install_size / 1024 / 1024:.02f} MiB') @@ -573,21 +638,25 @@ class LegendaryCLI: logger.info(f'Download size: {analysis.dl_size / 1024 / 1024:.02f} MiB ' f'(Compression savings: {compression:.01f}%)') logger.info(f'Reusable size: {analysis.reuse_size / 1024 / 1024:.02f} MiB (chunks) / ' - f'{analysis.unchanged / 1024 / 1024:.02f} MiB (unchanged)') + f'{analysis.unchanged / 1024 / 1024:.02f} MiB (unchanged / skipped)') - res = self.core.check_installation_conditions(analysis=analysis, install=igame) + res = self.core.check_installation_conditions(analysis=analysis, install=igame, game=game, + updating=self.core.is_installed(args.app_name), + ignore_space_req=args.ignore_space) - if res.failures: - logger.fatal('Download cannot proceed, the following errors occured:') - for msg in sorted(res.failures): - logger.fatal(msg) - exit(1) + if res.warnings or res.failures: + logger.info('Installation requirements check returned the following results:') if res.warnings: - logger.warning('Installation requirements check returned the following warnings:') for warn in sorted(res.warnings): logger.warning(warn) + if res.failures: + for msg in sorted(res.failures): + logger.fatal(msg) + logger.error('Installation cannot proceed, exiting.') + exit(1) + logger.info('Downloads are resumable, you can interrupt the download with ' 'CTRL-C and resume it using the same command later on.') @@ -608,7 +677,7 @@ class LegendaryCLI: except Exception as e: end_t = time.time() logger.info(f'Installation failed after {end_t - start_t:.02f} seconds.') - logger.warning(f'The following exception occured while waiting for the donlowader to finish: {e!r}. ' + logger.warning(f'The following exception occurred while waiting for the downloader to finish: {e!r}. ' f'Try restarting the process, the resume file will be used to start where it failed. ' f'If it continues to fail please open an issue on GitHub.') else: @@ -649,15 +718,22 @@ class LegendaryCLI: logger.info('This game supports cloud saves, syncing is handled by the "sync-saves" command.') logger.info(f'To download saves for this game run "legendary sync-saves {args.app_name}"') - if args.repair_mode and os.path.exists(repair_file): - igame = self.core.get_installed_game(game.app_name) - if igame.needs_verification: - igame.needs_verification = False - self.core.install_game(igame) + old_igame = self.core.get_installed_game(game.app_name) + if old_igame and args.repair_mode and os.path.exists(repair_file): + if old_igame.needs_verification: + old_igame.needs_verification = False + self.core.install_game(old_igame) logger.debug('Removing repair file.') os.remove(repair_file) + # check if install tags have changed, if they did; try deleting files that are no longer required. + if old_igame and old_igame.install_tags != igame.install_tags: + old_igame.install_tags = igame.install_tags + self.logger.info('Deleting now untagged files.') + self.core.uninstall_tag(old_igame) + self.core.install_game(old_igame) + logger.info(f'Finished installation process in {end_t - start_t:.02f} seconds.') def _handle_postinstall(self, postinstall, igame, yes=False): @@ -703,10 +779,10 @@ class LegendaryCLI: for dlc in dlcs: if (idlc := self.core.get_installed_game(dlc.app_name)) is not None: logger.info(f'Uninstalling DLC "{dlc.app_name}"...') - self.core.uninstall_game(idlc) + self.core.uninstall_game(idlc, delete_files=not args.keep_files) logger.info(f'Removing "{igame.title}" from "{igame.install_path}"...') - self.core.uninstall_game(igame, delete_root_directory=True) + self.core.uninstall_game(igame, delete_files=not args.keep_files, delete_root_directory=True) logger.info('Game has been uninstalled.') except Exception as e: logger.warning(f'Removing game failed: {e!r}, please remove {igame.install_path} manually.') @@ -769,6 +845,9 @@ class LegendaryCLI: logger.info(f'Run "legendary repair {args.app_name}" to repair your game installation.') def import_game(self, args): + # make sure path is absolute + args.app_path = os.path.abspath(args.app_path) + if not os.path.exists(args.app_path): logger.error(f'Specified path "{args.app_path}" does not exist!') exit(1) @@ -813,7 +892,7 @@ class LegendaryCLI: f'with legendary. Run "legendary repair {args.app_name}" to do so.') else: logger.info(f'Installation had Epic Games Launcher metadata for version "{igame.version}", ' - f'verification will not be requried.') + f'verification will not be required.') logger.info('Game has been imported.') def egs_sync(self, args): @@ -834,6 +913,11 @@ class LegendaryCLI: self.core.lgd.config.remove_option('Legendary', 'egl_sync') return + if not self.core.lgd.assets: + logger.error('Legendary is missing game metadata, please login (if not already) and use the ' + '"status" command to fetch necessary information to set-up syncing.') + return + if not self.core.egl.programdata_path: if not args.egl_manifest_path and not args.egl_wine_prefix: # search default Lutris install path @@ -904,6 +988,9 @@ class LegendaryCLI: for egl_game in importable: print(' *', egl_game.app_name, '-', egl_game.display_name) + print('\nNote: Only games that are also in Legendary\'s database are listed, ' + 'if anything is missing run "list-games" first to update it.') + if args.yes or get_boolean_choice('Do you want to import the games from EGL?'): for egl_game in importable: logger.info(f'Importing "{egl_game.display_name}"...') @@ -936,15 +1023,67 @@ class LegendaryCLI: else: self.core.egl_sync() + def status(self, args): + if not args.offline: + try: + if not self.core.login(): + logger.error('Log in failed!') + exit(1) + except ValueError: + pass + + if not self.core.lgd.userdata: + user_name = '' + args.offline = True + else: + user_name = self.core.lgd.userdata['displayName'] + + games_available = len(self.core.get_game_list(update_assets=not args.offline)) + games_installed = len(self.core.get_installed_list()) + if args.json: + print(json.dumps(dict( + account=user_name, + games_available=games_available, + games_installed=games_installed, + egl_sync_enabled=self.core.egl_sync_enabled, + config_directory=self.core.lgd.path + ), indent=2, sort_keys=True)) + return + + print(f'Epic account: {user_name}') + print(f'Games available: {games_available}') + print(f'Games installed: {games_installed}') + print(f'EGL Sync enabled: {self.core.egl_sync_enabled}') + print(f'Config directory: {self.core.lgd.path}') + + def cleanup(self, args): + before = self.core.lgd.get_dir_size() + # delete metadata + logger.debug('Removing app metadata...') + app_names = set(g.app_name for g in self.core.get_assets(update_assets=False)) + self.core.lgd.clean_metadata(app_names) + + if not args.keep_manifests: + logger.debug('Removing manifests...') + installed = [(ig.app_name, ig.version) for ig in self.core.get_installed_list()] + installed.extend((ig.app_name, ig.version) for ig in self.core.get_installed_dlc_list()) + self.core.lgd.clean_manifests(installed) + + logger.debug('Removing tmp data') + self.core.lgd.clean_tmp_data() + + after = self.core.lgd.get_dir_size() + logger.info(f'Cleanup complete! Removed {(before - after)/1024/1024:.02f} MiB.') + def main(): parser = argparse.ArgumentParser(description=f'Legendary v{__version__} - "{__codename__}"') parser.register('action', 'parsers', AliasedSubParsersAction) # general arguments - parser.add_argument('-v', dest='debug', action='store_true', help='Set loglevel to debug') + parser.add_argument('-v', '--debug', dest='debug', action='store_true', help='Set loglevel to debug') parser.add_argument('-y', '--yes', dest='yes', action='store_true', help='Default to yes for all prompts') - parser.add_argument('-V', dest='version', action='store_true', help='Print version and exit') + parser.add_argument('-V', '--version', dest='version', action='store_true', help='Print version and exit') # all the commands subparsers = parser.add_subparsers(title='Commands', dest='subparser_name') @@ -965,6 +1104,8 @@ def main(): verify_parser = subparsers.add_parser('verify-game', help='Verify a game\'s local files') import_parser = subparsers.add_parser('import-game', help='Import an already installed game') egl_sync_parser = subparsers.add_parser('egl-sync', help='Setup or run Epic Games Launcher sync') + status_parser = subparsers.add_parser('status', help='Show legendary status information') + clean_parser = subparsers.add_parser('cleanup', help='Remove old temporary, metadata, and manifest files') install_parser.add_argument('app_name', help='Name of the app', metavar='') uninstall_parser.add_argument('app_name', help='Name of the app', metavar='') @@ -1003,6 +1144,8 @@ def main(): help='Manifest URL or path to use instead of the CDN one (e.g. for downgrading)') install_parser.add_argument('--old-manifest', dest='override_old_manifest', action='store', metavar='', help='Manifest URL or path to use as the old one (e.g. for testing patching)') + install_parser.add_argument('--delta-manifest', dest='override_delta_manifest', action='store', metavar='', + help='Manifest URL or path to use as the delta one (e.g. for testing)') install_parser.add_argument('--base-url', dest='override_base_url', action='store', metavar='', help='Base URL to download from (e.g. to test or switch to a different CDNs)') install_parser.add_argument('--force', dest='force', action='store_true', @@ -1010,7 +1153,7 @@ def main(): install_parser.add_argument('--disable-patching', dest='disable_patching', action='store_true', help='Do not attempt to patch existing installation (download entire changed files)') install_parser.add_argument('--download-only', '--no-install', dest='no_install', action='store_true', - help='Do not intall app and do not run prerequisite installers after download') + help='Do not install app and do not run prerequisite installers after download') install_parser.add_argument('--update-only', dest='update_only', action='store_true', help='Only update, do not do anything if specified app is not installed') install_parser.add_argument('--dlm-debug', dest='dlm_debug', action='store_true', @@ -1032,6 +1175,17 @@ def main(): help='Set save game path to be used for sync-saves') install_parser.add_argument('--repair', dest='repair_mode', action='store_true', help='Repair installed game by checking and redownloading corrupted/missing files') + install_parser.add_argument('--repair-and-update', dest='repair_and_update', action='store_true', + help='Update game to the latest version when repairing') + install_parser.add_argument('--ignore-free-space', dest='ignore_space', action='store_true', + help='Do not abort if not enough free space is available') + install_parser.add_argument('--disable-delta-manifests', dest='disable_delta', action='store_true', + help='Do not use delta manifests when updating (may increase download size)') + install_parser.add_argument('--reset-sdl', dest='reset_sdl', action='store_true', + help='Reset selective downloading choices (requires repair to download new components)') + + uninstall_parser.add_argument('--keep-files', dest='keep_files', action='store_true', + help='Keep files but remove game from Legendary database') launch_parser.add_argument('--offline', dest='offline', action='store_true', default=False, help='Skip login and launch game without online authentication') @@ -1050,6 +1204,8 @@ def main(): help='Save parameters used to launch to config (does not include env vars)') launch_parser.add_argument('--reset-defaults', dest='reset_defaults', action='store_true', help='Reset config settings for app and exit') + launch_parser.add_argument('--override-exe', dest='executable_override', action='store', metavar='', + help='Override executable to launch (relative path)') if os.name != 'nt': launch_parser.add_argument('--wine', dest='wine_bin', action='store', metavar='', @@ -1074,6 +1230,7 @@ def main(): help='Also include Unreal Engine content (Engine/Marketplace) in list') list_parser.add_argument('--csv', dest='csv', action='store_true', help='List games in CSV format') list_parser.add_argument('--tsv', dest='tsv', action='store_true', help='List games in TSV format') + list_parser.add_argument('--json', dest='json', action='store_true', help='List games in JSON format') list_installed_parser.add_argument('--check-updates', dest='check_updates', action='store_true', help='Check for updates for installed games') @@ -1081,6 +1238,8 @@ def main(): help='List games in CSV format') list_installed_parser.add_argument('--tsv', dest='tsv', action='store_true', help='List games in TSV format') + list_installed_parser.add_argument('--json', dest='json', action='store_true', + help='List games in JSON format') list_installed_parser.add_argument('--show-dirs', dest='include_dir', action='store_true', help='Print installation directory in output') @@ -1092,6 +1251,7 @@ def main(): help='Manifest URL or path to use instead of the CDN one') list_files_parser.add_argument('--csv', dest='csv', action='store_true', help='Output in CSV format') list_files_parser.add_argument('--tsv', dest='tsv', action='store_true', help='Output in TSV format') + list_files_parser.add_argument('--json', dest='json', action='store_true', help='Output in JSON format') list_files_parser.add_argument('--hashlist', dest='hashlist', action='store_true', help='Output file hash list in hashcheck/sha1sum -c compatible format') list_files_parser.add_argument('--install-tag', dest='install_tag', action='store', metavar='', @@ -1132,6 +1292,14 @@ def main(): egl_sync_parser.add_argument('--unlink', dest='unlink', action='store_true', help='Disable sync and remove EGL metadata from installed games') + status_parser.add_argument('--offline', dest='offline', action='store_true', + help='Only print offline status information, do not login') + status_parser.add_argument('--json', dest='json', action='store_true', + help='Show status in JSON format') + + clean_parser.add_argument('--keep-manifests', dest='keep_manifests', action='store_true', + help='Do not delete old manifests') + args, extra = parser.parse_known_args() if args.version: @@ -1141,7 +1309,7 @@ def main(): if args.subparser_name not in ('auth', 'list-games', 'list-installed', 'list-files', 'launch', 'download', 'uninstall', 'install', 'update', 'repair', 'list-saves', 'download-saves', 'sync-saves', - 'verify-game', 'import-game', 'egl-sync'): + 'verify-game', 'import-game', 'egl-sync', 'status', 'cleanup'): print(parser.format_help()) # Print the main help *and* the help for all of the subcommands. Thanks stackoverflow! @@ -1164,10 +1332,10 @@ def main(): logging.getLogger('requests').setLevel(logging.WARNING) logging.getLogger('urllib3').setLevel(logging.WARNING) - # -y having to be specified before the subcommand is a little counter-intuitive - # For now show a warning if a user is misusing that flag + # if --yes is used as part of the subparsers arguments manually set the flag in the main parser. if '-y' in extra or '--yes' in extra: - logger.warning('-y/--yes flag needs to be specified *before* the command name') + args.yes = True + extra = [i for i in extra if i not in ('--yes', '-y')] # technically args.func() with setdefaults could work (see docs on subparsers) # but that would require all funcs to accept args and extra... @@ -1198,6 +1366,10 @@ def main(): cli.import_game(args) elif args.subparser_name == 'egl-sync': cli.egs_sync(args) + elif args.subparser_name == 'status': + cli.status(args) + elif args.subparser_name == 'cleanup': + cli.cleanup(args) except KeyboardInterrupt: logger.info('Command was aborted via KeyboardInterrupt, cleaning up...') diff --git a/legendary/core.py b/legendary/core.py index fc541ef..0c5e212 100644 --- a/legendary/core.py +++ b/legendary/core.py @@ -12,7 +12,7 @@ from datetime import datetime, timezone from locale import getdefaultlocale from multiprocessing import Queue from random import choice as randchoice -from requests import Request, session +from requests import session from requests.exceptions import HTTPError from typing import List, Dict from uuid import uuid4 @@ -31,6 +31,8 @@ from legendary.models.manifest import Manifest, ManifestMeta from legendary.models.chunk import Chunk from legendary.utils.game_workarounds import is_opt_enabled from legendary.utils.savegame_helper import SaveGameHelper +from legendary.utils.manifests import combine_manifests +from legendary.utils.wine_helpers import read_registry, get_shell_folders # ToDo: instead of true/false return values for success/failure actually raise an exception that the CLI/GUI @@ -63,17 +65,12 @@ class LegendaryCore: self.local_timezone = datetime.now().astimezone().tzinfo self.language_code, self.country_code = ('en', 'US') - def get_locale(self): - locale = self.lgd.config.get('Legendary', 'locale', fallback=getdefaultlocale()[0]) - - if locale: + if locale := self.lgd.config.get('Legendary', 'locale', fallback=getdefaultlocale()[0]): try: self.language_code, self.country_code = locale.split('-' if '-' in locale else '_') self.log.debug(f'Set locale to {self.language_code}-{self.country_code}') - - # if egs is loaded make sure to override its language setting as well - if self.egs: - self.egs.language_code, self.egs.country_code = self.language_code, self.country_code + # adjust egs api language as well + self.egs.language_code, self.egs.country_code = self.language_code, self.country_code except Exception as e: self.log.warning(f'Getting locale failed: {e!r}, falling back to using en-US.') else: @@ -103,9 +100,9 @@ class LegendaryCore: 'X-Requested-With': 'XMLHttpRequest', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' - 'EpicGamesLauncher/10.16.1-13343695+++Portal+Release-Live ' - 'UnrealEngine/4.23.0-13343695+++Portal+Release-Live ' - 'Chrome/59.0.3071.15 Safari/537.36' + 'EpicGamesLauncher/11.0.1-14907503+++Portal+Release-Live ' + 'UnrealEngine/4.23.0-14907503+++Portal+Release-Live ' + 'Chrome/84.0.4147.38 Safari/537.36' }) # get first set of cookies (EPIC_BEARER_TOKEN etc.) @@ -190,12 +187,15 @@ class LegendaryCore: return True def get_assets(self, update_assets=False, platform_override=None) -> List[GameAsset]: - # do not save and always fetch list when platform is overriden + # do not save and always fetch list when platform is overridden if platform_override: return [GameAsset.from_egs_json(a) for a in self.egs.get_game_assets(platform=platform_override)] if not self.lgd.assets or update_assets: + # if not logged in, return empty list + if not self.egs.user: + return [] self.lgd.assets = [GameAsset.from_egs_json(a) for a in self.egs.get_game_assets()] return self.lgd.assets @@ -204,7 +204,13 @@ class LegendaryCore: if update: self.get_assets(update_assets=True) - return next(i for i in self.lgd.assets if i.app_name == app_name) + try: + return next(i for i in self.lgd.assets if i.app_name == app_name) + except StopIteration: + raise ValueError + + def asset_valid(self, app_name) -> bool: + return any(i.app_name == app_name for i in self.lgd.assets) def get_game(self, app_name, update_meta=False) -> Game: if update_meta: @@ -217,8 +223,6 @@ class LegendaryCore: def get_game_and_dlc_list(self, update_assets=True, platform_override=None, skip_ue=True) -> (List[Game], Dict[str, Game]): - # resolve locale - self.get_locale() _ret = [] _dlc = defaultdict(list) @@ -247,13 +251,17 @@ class LegendaryCore: if game.is_dlc: _dlc[game.metadata['mainGameItem']['id']].append(game) - else: + elif not any(i['path'] == 'mods' for i in game.metadata.get('categories', [])): _ret.append(game) return _ret, _dlc def get_dlc_for_game(self, app_name): game = self.get_game(app_name) + if not game: + self.log.warning(f'Metadata for {app_name} is missing!') + return [] + if game.is_dlc: # dlc shouldn't have DLC return [] @@ -288,7 +296,8 @@ class LegendaryCore: user: str = None, extra_args: list = None, wine_bin: str = None, wine_pfx: str = None, language: str = None, wrapper: str = None, - disable_wine: bool = False) -> (list, str, dict): + disable_wine: bool = False, + executable_override: str = None) -> (list, str, dict): install = self.lgd.get_installed_game(app_name) game = self.lgd.get_game_meta(app_name) @@ -304,13 +313,22 @@ class LegendaryCore: if user: user_name = user - game_exe = os.path.join(install.install_path, - install.executable.replace('\\', '/').lstrip('/')) + if executable_override or (executable_override := self.lgd.config.get(app_name, 'override_exe', fallback=None)): + game_exe = os.path.join(install.install_path, + executable_override.replace('\\', '/')) + if not os.path.exists(game_exe): + raise ValueError(f'Executable path is invalid: {game_exe}') + else: + game_exe = os.path.join(install.install_path, + install.executable.replace('\\', '/').lstrip('/')) + working_dir = os.path.split(game_exe)[0] params = [] - if wrapper or (wrapper := self.lgd.config.get(app_name, 'wrapper', fallback=None)): + if wrapper or (wrapper := self.lgd.config.get(app_name, 'wrapper', + fallback=self.lgd.config.get('default', 'wrapper', + fallback=None))): params.extend(shlex.split(wrapper)) if os.name != 'nt' and not disable_wine: @@ -320,13 +338,14 @@ class LegendaryCore: # check if there's a game specific override wine_bin = self.lgd.config.get(app_name, 'wine_executable', fallback=wine_bin) - if not self.lgd.config.getboolean(app_name, 'no_wine', fallback=False): + if not self.lgd.config.getboolean(app_name, 'no_wine', + fallback=self.lgd.config.get('default', 'no_wine', fallback=False)): params.append(wine_bin) params.append(game_exe) if install.launch_parameters: - params.extend(shlex.split(install.launch_parameters)) + params.extend(shlex.split(install.launch_parameters, posix=False)) params.extend([ '-AUTH_LOGIN=unused', @@ -347,7 +366,6 @@ class LegendaryCore: language_code = self.lgd.config.get(app_name, 'language', fallback=language) if not language_code: # fall back to system or config language - self.get_locale() language_code = self.language_code params.extend([ @@ -365,10 +383,10 @@ class LegendaryCore: # get environment overrides from config env = os.environ.copy() + if 'default.env' in self.lgd.config: + env.update({k: v for k, v in self.lgd.config[f'default.env'].items() if v and not k.startswith(';')}) if f'{app_name}.env' in self.lgd.config: - env.update(dict(self.lgd.config[f'{app_name}.env'])) - elif 'default.env' in self.lgd.config: - env.update(dict(self.lgd.config['default.env'])) + env.update({k: v for k, v in self.lgd.config[f'{app_name}.env'].items() if v and not k.startswith(';')}) if wine_pfx: env['WINEPREFIX'] = wine_pfx @@ -403,19 +421,45 @@ class LegendaryCore: # the following variables are known: path_vars = { - '{appdata}': os.path.expandvars('%APPDATA%'), '{installdir}': igame.install_path, - '{userdir}': os.path.expandvars('%userprofile%/documents'), '{epicid}': self.lgd.userdata['account_id'] } - # the following variables are in the EGL binary but are not used by any of - # my games and I'm not sure where they actually point at: - # {UserProfile} (Probably %USERPROFILE%) - # {UserSavedGames} + + if os.name == 'nt': + path_vars.update({ + '{appdata}': os.path.expandvars('%APPDATA%'), + '{userdir}': os.path.expandvars('%userprofile%/documents'), + # '{userprofile}': os.path.expandvars('%userprofile%'), # possibly wrong + '{usersavedgames}': os.path.expandvars('%userprofile%/Saved Games') + }) + else: + # attempt to get WINE prefix from config + wine_pfx = self.lgd.config.get(app_name, 'wine_prefix', fallback=None) + if not wine_pfx: + wine_pfx = self.lgd.config.get(f'{app_name}.env', 'WINEPREFIX', fallback=None) + if not wine_pfx: + proton_pfx = self.lgd.config.get(f'{app_name}.env', 'STEAM_COMPAT_DATA_PATH', fallback=None) + if proton_pfx: + wine_pfx = f'{proton_pfx}/pfx' + if not wine_pfx: + wine_pfx = os.path.expanduser('~/.wine') + + # if we have a prefix, read the `user.reg` file and get the proper paths. + if os.path.isdir(wine_pfx): + wine_reg = read_registry(wine_pfx) + wine_folders = get_shell_folders(wine_reg, wine_pfx) + # path_vars['{userprofile}'] = user_path + path_vars['{appdata}'] = wine_folders['AppData'] + # this maps to ~/Documents, but the name is locale-dependent so just resolve the symlink from WINE + path_vars['{userdir}'] = os.path.realpath(wine_folders['Personal']) + path_vars['{usersavedgames}'] = wine_folders['{4C5C32FF-BB9D-43B0-B5B4-2D72E54EAAA4}'] + + # replace backslashes + save_path = save_path.replace('\\', '/') # these paths should always use a forward slash new_save_path = [path_vars.get(p.lower(), p) for p in save_path.split('/')] - return os.path.join(*new_save_path) + return os.path.realpath(os.path.join(*new_save_path)) def check_savegame_state(self, path: str, save: SaveGameFile) -> (SaveGameStatus, (datetime, datetime)): latest = 0 @@ -513,6 +557,11 @@ class LegendaryCore: if r.status_code != 200: self.log.error(f'Download failed, status code: {r.status_code}') continue + + if not r.content: + self.log.error('Manifest is empty! Skipping...') + continue + m = self.load_manifest(r.content) # download chunks required for extraction @@ -607,12 +656,11 @@ class LegendaryCore: if base_url not in base_urls: base_urls.append(base_url) - params = None if 'queryParams' in manifest: - params = {p['name']: p['value'] for p in manifest['queryParams']} - - # build url with a prepared request - manifest_urls.append(Request('GET', manifest['uri'], params=params).prepare().url) + params = '&'.join(f'{p["name"]}={p["value"]}' for p in manifest['queryParams']) + manifest_urls.append(f'{manifest["uri"]}?{params}') + else: + manifest_urls.append(manifest['uri']) return manifest_urls, base_urls @@ -636,6 +684,17 @@ class LegendaryCore: return new_manifest_data, base_urls + def get_delta_manifest(self, base_url, old_build_id, new_build_id): + """Get optimized delta manifest (doesn't seem to exist for most games)""" + if old_build_id == new_build_id: + return None + + r = self.egs.unauth_session.get(f'{base_url}/Deltas/{new_build_id}/{old_build_id}.delta') + if r.status_code == 200: + return r.content + else: + return None + def prepare_download(self, game: Game, base_game: Game = None, base_path: str = '', status_q: Queue = None, max_shm: int = 0, max_workers: int = 0, force: bool = False, disable_patching: bool = False, @@ -644,8 +703,9 @@ class LegendaryCore: platform_override: str = '', file_prefix_filter: list = None, file_exclude_filter: list = None, file_install_tag: list = None, dl_optimizations: bool = False, dl_timeout: int = 10, - repair: bool = False, egl_guid: str = '' - ) -> (DLManager, AnalysisResult, ManifestMeta): + repair: bool = False, repair_use_latest: bool = False, + disable_delta: bool = False, override_delta_manifest: str = '', + egl_guid: str = '') -> (DLManager, AnalysisResult, ManifestMeta): # load old manifest old_manifest = None @@ -682,11 +742,31 @@ class LegendaryCore: self.log.info('Parsing game manifest...') new_manifest = self.load_manifest(new_manifest_data) self.log.debug(f'Base urls: {base_urls}') - self.lgd.save_manifest(game.app_name, new_manifest_data) # save manifest with version name as well for testing/downgrading/etc. self.lgd.save_manifest(game.app_name, new_manifest_data, version=new_manifest.meta.build_version) + # check if we should use a delta manifest or not + disable_delta = disable_delta or ((override_old_manifest or override_manifest) and not override_delta_manifest) + if old_manifest and new_manifest: + disable_delta = disable_delta or (old_manifest.meta.build_id == new_manifest.meta.build_id) + if old_manifest and new_manifest and not disable_delta: + if override_delta_manifest: + self.log.info(f'Overriding delta manifest with "{override_delta_manifest}"') + delta_manifest_data, _ = self.get_uri_manifest(override_delta_manifest) + else: + delta_manifest_data = self.get_delta_manifest(randchoice(base_urls), + old_manifest.meta.build_id, + new_manifest.meta.build_id) + if delta_manifest_data: + delta_manifest = self.load_manifest(delta_manifest_data) + self.log.info(f'Using optimized delta manifest to upgrade from build ' + f'"{old_manifest.meta.build_id}" to ' + f'"{new_manifest.meta.build_id}"...') + combine_manifests(new_manifest, delta_manifest) + else: + self.log.debug(f'No Delta manifest received from CDN.') + # reuse existing installation's directory if igame := self.get_installed_game(base_game.app_name if base_game else game.app_name): install_path = igame.install_path @@ -715,9 +795,11 @@ class LegendaryCore: self.log.info(f'Install path: {install_path}') if repair: - # use installed manifest for repairs, do not update to latest version (for now) - new_manifest = old_manifest - old_manifest = None + if not repair_use_latest: + # use installed manifest for repairs instead of updating + new_manifest = old_manifest + old_manifest = None + filename = clean_filename(f'{game.app_name}.repair') resume_file = os.path.join(self.lgd.get_tmp_path(), filename) force = False @@ -737,14 +819,17 @@ class LegendaryCore: self.log.debug(f'Using base URL: {base_url}') if not max_shm: - max_shm = self.lgd.config.getint('Legendary', 'max_memory', fallback=1024) + max_shm = self.lgd.config.getint('Legendary', 'max_memory', fallback=2048) - if dl_optimizations or is_opt_enabled(game.app_name): + if dl_optimizations or is_opt_enabled(game.app_name, new_manifest.meta.build_version): self.log.info('Download order optimizations are enabled.') process_opt = True else: process_opt = False + if not max_workers: + max_workers = self.lgd.config.getint('Legendary', 'max_workers', fallback=0) + dlm = DLManager(install_path, base_url, resume_file=resume_file, status_q=status_q, max_shared_memory=max_shm * 1024 * 1024, max_workers=max_workers, dl_timeout=dl_timeout) @@ -770,12 +855,16 @@ class LegendaryCore: launch_parameters=new_manifest.meta.launch_command, can_run_offline=offline == 'true', requires_ot=ot == 'true', is_dlc=base_game is not None, install_size=anlres.install_size, - egl_guid=egl_guid) + egl_guid=egl_guid, install_tags=file_install_tag) return dlm, anlres, igame @staticmethod - def check_installation_conditions(analysis: AnalysisResult, install: InstalledGame) -> ConditionCheckResult: + def check_installation_conditions(analysis: AnalysisResult, + install: InstalledGame, + game: Game, + updating: bool = False, + ignore_space_req: bool = False) -> ConditionCheckResult: results = ConditionCheckResult(failures=set(), warnings=set()) # if on linux, check for eac in the files @@ -797,12 +886,42 @@ class LegendaryCore: results.warnings.add('This game is not marked for offline use (may still work).') # check if enough disk space is free (dl size is the approximate amount the installation will grow) - min_disk_space = analysis.uncompressed_dl_size + analysis.biggest_file_size + min_disk_space = analysis.install_size + if updating: + min_disk_space += analysis.biggest_file_size + + # todo when resuming, only check remaining files _, _, free = shutil.disk_usage(os.path.split(install.install_path)[0]) if free < min_disk_space: free_mib = free / 1024 / 1024 required_mib = min_disk_space / 1024 / 1024 - results.failures.add(f'Not enough available disk space! {free_mib:.02f} MiB < {required_mib:.02f} MiB') + if ignore_space_req: + results.warnings.add(f'Potentially not enough available disk space! ' + f'{free_mib:.02f} MiB < {required_mib:.02f} MiB') + else: + results.failures.add(f'Not enough available disk space! ' + f'{free_mib:.02f} MiB < {required_mib:.02f} MiB') + + # check if the game actually ships the files or just a uplay installer + packed game files + executables = [f for f in analysis.manifest_comparison.added if + f.lower().endswith('.exe') and not f.startswith('Installer/')] + if not updating and not any('uplay' not in e.lower() for e in executables) and \ + any('uplay' in e.lower() for e in executables): + results.failures.add('This game requires installation via Uplay and does not ship executable game files.') + + # check if the game launches via uplay + if install.executable == 'UplayLaunch.exe': + results.warnings.add('This game requires launching via Uplay, it is recommended to install the game ' + 'via Uplay instead.') + + # check if the game requires linking to an external account first + partner_link = game.metadata.get('customAttributes', {}).get('partnerLinkType', {}).get('value', None) + if partner_link == 'ubisoft': + results.warnings.add('This game requires linking to and activating on a Ubisoft account first, ' + 'this is not currently supported.') + elif partner_link: + results.warnings.add(f'This game requires linking to "{partner_link}", ' + f'this is currently unsupported and the game may not work.') return results @@ -843,6 +962,22 @@ class LegendaryCore: self.lgd.remove_installed_game(installed_game.app_name) + def uninstall_tag(self, installed_game: InstalledGame): + manifest = self.load_manifest(self.get_installed_manifest(installed_game.app_name)[0]) + tags = installed_game.install_tags + if '' not in tags: + tags.append('') + + # Create list of files that are now no longer needed *and* actually exist on disk + filelist = [ + fm.filename for fm in manifest.file_manifest_list.elements if + not any(((fit in fm.install_tags) or (not fit and not fm.install_tags)) for fit in tags) + and os.path.exists(os.path.join(installed_game.install_path, fm.filename)) + ] + + if not delete_filelist(installed_game.install_path, filelist): + self.log.warning(f'Deleting some deselected files failed, please check/remove manually.') + def prereq_installed(self, app_name): igame = self.lgd.get_installed_game(app_name) igame.prereq_info['installed'] = True @@ -872,7 +1007,7 @@ class LegendaryCore: if mf and os.path.exists(os.path.join(app_path, '.egstore', mf)): manifest_data = open(os.path.join(app_path, '.egstore', mf), 'rb').read() else: - self.log.warning('.egstore folder exists but manifest file is missing, contiuing as regular import...') + self.log.warning('.egstore folder exists but manifest file is missing, continuing as regular import...') # If there's no in-progress installation assume the game doesn't need to be verified if mf and not os.path.exists(os.path.join(app_path, '.egstore', 'bps')): @@ -896,7 +1031,6 @@ class LegendaryCore: # parse and save manifest to disk for verification step of import new_manifest = self.load_manifest(manifest_data) - self.lgd.save_manifest(game.app_name, manifest_data) self.lgd.save_manifest(game.app_name, manifest_data, version=new_manifest.meta.build_version) install_size = sum(fm.file_size for fm in new_manifest.file_manifest_list.elements) @@ -918,7 +1052,9 @@ class LegendaryCore: def egl_get_importable(self): return [g for g in self.egl.get_manifests() - if not self.is_installed(g.app_name) and g.main_game_appname == g.app_name] + if not self.is_installed(g.app_name) and + g.main_game_appname == g.app_name and + self.asset_valid(g.app_name)] def egl_get_exportable(self): if not self.egl.manifests: @@ -926,6 +1062,9 @@ class LegendaryCore: return [g for g in self.get_installed_list() if g.app_name not in self.egl.manifests] def egl_import(self, app_name): + if not self.asset_valid(app_name): + raise ValueError(f'To-be-imported game {app_name} not in game asset database!') + self.log.debug(f'Importing "{app_name}" from EGL') # load egl json file try: @@ -962,9 +1101,15 @@ class LegendaryCore: with open(manifest_filename, 'rb') as f: manifest_data = f.read() new_manifest = self.load_manifest(manifest_data) - self.lgd.save_manifest(lgd_igame.app_name, manifest_data) self.lgd.save_manifest(lgd_igame.app_name, manifest_data, version=new_manifest.meta.build_version) + + # transfer install tag choices to config + if lgd_igame.install_tags: + if app_name not in self.lgd.config: + self.lgd.config[app_name] = dict() + self.lgd.config.set(app_name, 'install_tags', ','.join(lgd_igame.install_tags)) + # mark game as installed _ = self._install_game(lgd_igame) return @@ -1039,7 +1184,8 @@ class LegendaryCore: return self.egl_restore_or_uninstall(lgd_igame) else: egl_igame = self.egl.get_manifest(app_name) - if egl_igame.app_version_string != lgd_igame.version: + if (egl_igame.app_version_string != lgd_igame.version) or \ + (egl_igame.install_tags != lgd_igame.install_tags): self.log.info(f'App "{egl_igame.app_name}" has been updated from EGL, syncing...') return self.egl_import(egl_igame.app_name) else: @@ -1047,12 +1193,15 @@ class LegendaryCore: for egl_igame in self.egl.get_manifests(): if egl_igame.main_game_appname != egl_igame.app_name: # skip DLC continue + if not self.asset_valid(egl_igame.app_name): # skip non-owned games + continue if not self._is_installed(egl_igame.app_name): self.egl_import(egl_igame.app_name) else: lgd_igame = self._get_installed_game(egl_igame.app_name) - if lgd_igame.version != egl_igame.app_version_string: + if (egl_igame.app_version_string != lgd_igame.version) or \ + (egl_igame.install_tags != lgd_igame.install_tags): self.log.info(f'App "{egl_igame.app_name}" has been updated from EGL, syncing...') self.egl_import(egl_igame.app_name) @@ -1075,4 +1224,3 @@ class LegendaryCore: Do cleanup, config saving, and exit. """ self.lgd.save_config() - diff --git a/legendary/downloader/manager.py b/legendary/downloader/manager.py index 6d41c6b..0b84d2a 100644 --- a/legendary/downloader/manager.py +++ b/legendary/downloader/manager.py @@ -137,18 +137,21 @@ class DLManager(Process): except Exception as e: self.log.warning(f'Reading resume file failed: {e!r}, continuing as normal...') - # Not entirely sure what install tags are used for, only some titles have them. - # Let's add it for testing anyway. - if file_install_tag: + # Install tags are used for selective downloading, e.g. for language packs + additional_deletion_tasks = [] + if file_install_tag is not None: if isinstance(file_install_tag, str): file_install_tag = [file_install_tag] files_to_skip = set(i.filename for i in manifest.file_manifest_list.elements - if not any(fit in i.install_tags for fit in file_install_tag)) + if not any((fit in i.install_tags) or (not fit and not i.install_tags) + for fit in file_install_tag)) self.log.info(f'Found {len(files_to_skip)} files to skip based on install tag.') mc.added -= files_to_skip mc.changed -= files_to_skip mc.unchanged |= files_to_skip + for fname in sorted(files_to_skip): + additional_deletion_tasks.append(FileTask(fname, delete=True, silent=True)) # if include/exclude prefix has been set: mark all files that are not to be downloaded as unchanged if file_exclude_filter: @@ -194,7 +197,7 @@ class DLManager(Process): analysis_res.unchanged = len(mc.unchanged) self.log.debug(f'{analysis_res.unchanged} unchanged files') - if processing_optimization and len(manifest.file_manifest_list.elements) > 8_000: + if processing_optimization and len(manifest.file_manifest_list.elements) > 100_000: self.log.warning('Manifest contains too many files, processing optimizations will be disabled.') processing_optimization = False elif processing_optimization: @@ -202,7 +205,6 @@ class DLManager(Process): # count references to chunks for determining runtime cache size later references = Counter() - file_to_chunks = defaultdict(set) fmlist = sorted(manifest.file_manifest_list.elements, key=lambda a: a.filename.lower()) @@ -216,54 +218,44 @@ class DLManager(Process): for cp in fm.chunk_parts: references[cp.guid_num] += 1 - if processing_optimization: - file_to_chunks[fm.filename].add(cp.guid_num) if processing_optimization: + s_time = time.time() # reorder the file manifest list to group files that share many chunks - # 5 is mostly arbitrary but has shown in testing to be a good choice + # 4 is mostly arbitrary but has shown in testing to be a good choice min_overlap = 4 - # enumerate the file list to try and find a "partner" for - # each file that shares the most chunks with it. - partners = dict() - filenames = [fm.filename for fm in fmlist] + # ignore files with less than N chunk parts, this speeds things up dramatically + cp_threshold = 5 - for num, filename in enumerate(filenames[:int((len(filenames) + 1) / 2)]): - chunks = file_to_chunks[filename] - partnerlist = list() - - for other_file in filenames[num + 1:]: - overlap = len(chunks & file_to_chunks[other_file]) - if overlap > min_overlap: - partnerlist.append(other_file) - - if not partnerlist: - continue - - partners[filename] = partnerlist - - # iterate over all the files again and this time around + remaining_files = {fm.filename: {cp.guid_num for cp in fm.chunk_parts} + for fm in fmlist if fm.filename not in mc.unchanged} _fmlist = [] - processed = set() - for fm in fmlist: - if fm.filename in processed: - continue - _fmlist.append(fm) - processed.add(fm.filename) - # try to find the file's "partner" - f_partners = partners.get(fm.filename, None) - if not f_partners: - continue - # add each partner to list at this point - for partner in f_partners: - if partner in processed: - continue - partner_fm = manifest.file_manifest_list.get_file_by_path(partner) - _fmlist.append(partner_fm) - processed.add(partner) + # iterate over all files that will be downloaded and pair up those that share the most chunks + for fm in fmlist: + if fm.filename not in remaining_files: + continue + + _fmlist.append(fm) + f_chunks = remaining_files.pop(fm.filename) + if len(f_chunks) < cp_threshold: + continue + + best_overlap, match = 0, None + for fname, chunks in remaining_files.items(): + if len(chunks) < cp_threshold: + continue + overlap = len(f_chunks & chunks) + if overlap > min_overlap and overlap > best_overlap: + best_overlap, match = overlap, fname + + if match: + _fmlist.append(manifest.file_manifest_list.get_file_by_path(match)) + remaining_files.pop(match) fmlist = _fmlist + opt_delta = time.time() - s_time + self.log.debug(f'Processing optimizations took {opt_delta:.01f} seconds.') # determine reusable chunks and prepare lookup table for reusable ones re_usable = defaultdict(dict) @@ -273,18 +265,21 @@ class DLManager(Process): old_file = old_manifest.file_manifest_list.get_file_by_path(changed) new_file = manifest.file_manifest_list.get_file_by_path(changed) - existing_chunks = dict() + existing_chunks = defaultdict(list) off = 0 for cp in old_file.chunk_parts: - existing_chunks[(cp.guid_num, cp.offset, cp.size)] = off + existing_chunks[cp.guid_num].append((off, cp.offset, cp.offset + cp.size)) off += cp.size for cp in new_file.chunk_parts: key = (cp.guid_num, cp.offset, cp.size) - if key in existing_chunks: - references[cp.guid_num] -= 1 - re_usable[changed][key] = existing_chunks[key] - analysis_res.reuse_size += cp.size + for file_o, cp_o, cp_end_o in existing_chunks[cp.guid_num]: + # check if new chunk part is wholly contained in the old chunk part + if cp_o <= cp.offset and (cp.offset + cp.size) <= cp_end_o: + references[cp.guid_num] -= 1 + re_usable[changed][key] = file_o + (cp.offset - cp_o) + analysis_res.reuse_size += cp.size + break last_cache_size = current_cache_size = 0 # set to determine whether a file is currently cached or not @@ -349,7 +344,7 @@ class DLManager(Process): self.tasks.append(FileTask(current_file.filename + u'.tmp', fopen=True)) self.tasks.extend(chunk_tasks) self.tasks.append(FileTask(current_file.filename + u'.tmp', close=True)) - # delete old file and rename temproary + # delete old file and rename temporary self.tasks.append(FileTask(current_file.filename, delete=True, rename=True, temporary_filename=current_file.filename + u'.tmp')) else: @@ -369,8 +364,17 @@ class DLManager(Process): if analysis_res.min_memory > self.max_shared_memory: shared_mib = f'{self.max_shared_memory / 1024 / 1024:.01f} MiB' required_mib = f'{analysis_res.min_memory / 1024 / 1024:.01f} MiB' - raise MemoryError(f'Current shared memory cache is smaller than required! {shared_mib} < {required_mib}. ' - f'Try running legendary with the --enable-reordering flag to reduce memory usage.') + suggested_mib = round(self.max_shared_memory / 1024 / 1024 + + (analysis_res.min_memory - self.max_shared_memory) / 1024 / 1024 + 32) + + if processing_optimization: + message = f'Try running legendary with "--enable-reordering --max-shared-memory {suggested_mib:.0f}"' + else: + message = 'Try running legendary with "--enable-reordering" to reduce memory usage, ' \ + f'or use "--max-shared-memory {suggested_mib:.0f}" to increase the limit.' + + raise MemoryError(f'Current shared memory cache is smaller than required: {shared_mib} < {required_mib}. ' + + message) # calculate actual dl and patch write size. analysis_res.dl_size = \ @@ -381,6 +385,7 @@ class DLManager(Process): # add jobs to remove files for fname in mc.removed: self.tasks.append(FileTask(fname, delete=True)) + self.tasks.extend(additional_deletion_tasks) analysis_res.num_chunks_cache = len(dl_cache_guids) self.chunk_data_list = manifest.chunk_data_list @@ -443,7 +448,7 @@ class DLManager(Process): old_filename=task.temporary_filename), timeout=1.0) elif task.delete: - self.writer_queue.put(WriterTask(task.filename, delete=True), timeout=1.0) + self.writer_queue.put(WriterTask(task.filename, delete=True, silent=task.silent), timeout=1.0) elif task.open: self.writer_queue.put(WriterTask(task.filename, fopen=True), timeout=1.0) current_file = task.filename @@ -524,6 +529,9 @@ class DLManager(Process): self.num_tasks_processed_since_last += 1 if res.closed and self.resume_file and res.success: + if res.filename.endswith('.tmp'): + res.filename = res.filename[:-4] + file_hash = self.hash_map[res.filename] # write last completed file to super simple resume file with open(self.resume_file, 'ab') as rf: diff --git a/legendary/downloader/workers.py b/legendary/downloader/workers.py index 6182ecf..8f34e83 100644 --- a/legendary/downloader/workers.py +++ b/legendary/downloader/workers.py @@ -22,7 +22,7 @@ class DLWorker(Process): self.o_q = out_queue self.session = requests.session() self.session.headers.update({ - 'User-Agent': 'EpicGamesLauncher/10.16.1-13343695+++Portal+Release-Live Windows/10.0.18363.1.256.64bit' + 'User-Agent': 'EpicGamesLauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit' }) self.max_retries = max_retries self.shm = SharedMemory(name=shm) @@ -217,7 +217,8 @@ class FileWorker(Process): try: os.remove(full_path) except OSError as e: - logger.error(f'Removing file failed: {e!r}') + if not j.silent: + logger.error(f'Removing file failed: {e!r}') self.o_q.put(WriterTaskResult(success=True, filename=j.filename)) continue diff --git a/legendary/lfs/lgndry.py b/legendary/lfs/lgndry.py index 13cbaa2..1008c1e 100644 --- a/legendary/lfs/lgndry.py +++ b/legendary/lfs/lgndry.py @@ -5,6 +5,8 @@ import os import configparser import logging +from pathlib import Path + from legendary.models.game import * from legendary.utils.lfs import clean_filename @@ -27,14 +29,41 @@ class LGDLFS: # EGS metadata self._game_metadata = dict() # Config with game specific settings (e.g. start parameters, env variables) - self.config = configparser.ConfigParser() + self.config = configparser.ConfigParser(comment_prefixes='/', allow_no_value=True) self.config.optionxform = str # ensure folders exist. - for f in ['', 'manifests', 'metadata', 'tmp', 'manifests/old']: + for f in ['', 'manifests', 'metadata', 'tmp']: if not os.path.exists(os.path.join(self.path, f)): os.makedirs(os.path.join(self.path, f)) + # if "old" folder exists migrate files and remove it + if os.path.exists(os.path.join(self.path, 'manifests', 'old')): + self.log.info('Migrating manifest files from old folders to new, please wait...') + # remove unversioned manifest files + for _f in os.listdir(os.path.join(self.path, 'manifests')): + if '.manifest' not in _f: + continue + if '_' not in _f or (_f.startswith('UE_') and _f.count('_') < 2): + self.log.debug(f'Deleting "{_f}" ...') + os.remove(os.path.join(self.path, 'manifests', _f)) + + # move files from "old" to the base folder + for _f in os.listdir(os.path.join(self.path, 'manifests', 'old')): + try: + self.log.debug(f'Renaming "{_f}"') + os.rename(os.path.join(self.path, 'manifests', 'old', _f), + os.path.join(self.path, 'manifests', _f)) + except Exception as e: + self.log.warning(f'Renaming manifest file "{_f}" failed: {e!r}') + + # remove "old" folder + try: + os.removedirs(os.path.join(self.path, 'manifests', 'old')) + except Exception as e: + self.log.warning(f'Removing "{os.path.join(self.path, "manifests", "old")}" folder failed: ' + f'{e!r}, please remove manually') + # try loading config self.config.read(os.path.join(self.path, 'config.ini')) # make sure "Legendary" section exists @@ -123,21 +152,17 @@ class LGDLFS: open(os.path.join(self.path, 'assets.json'), 'w'), indent=2, sort_keys=True) - def _get_manifest_filename(self, app_name, version=''): - if not version: - return os.path.join(self.path, 'manifests', f'{app_name}.manifest') - else: - # if a version is specified load it from the versioned directory - fname = clean_filename(f'{app_name}_{version}') - return os.path.join(self.path, 'manifests', 'old', f'{fname}.manifest') + def _get_manifest_filename(self, app_name, version): + fname = clean_filename(f'{app_name}_{version}') + return os.path.join(self.path, 'manifests', f'{fname}.manifest') - def load_manifest(self, app_name, version=''): + def load_manifest(self, app_name, version): try: return open(self._get_manifest_filename(app_name, version), 'rb').read() except FileNotFoundError: # all other errors should propagate return None - def save_manifest(self, app_name, manifest_data, version=''): + def save_manifest(self, app_name, manifest_data, version): with open(self._get_manifest_filename(app_name, version), 'wb') as f: f.write(manifest_data) @@ -172,6 +197,24 @@ class LGDLFS: except Exception as e: self.log.warning(f'Failed to delete file "{f}": {e!r}') + def clean_metadata(self, app_names): + for f in os.listdir(os.path.join(self.path, 'metadata')): + app_name = f.rpartition('.')[0] + if app_name not in app_names: + try: + os.remove(os.path.join(self.path, 'metadata', f)) + except Exception as e: + self.log.warning(f'Failed to delete file "{f}": {e!r}') + + def clean_manifests(self, in_use): + in_use_files = set(f'{clean_filename(f"{app_name}_{version}")}.manifest' for app_name, version in in_use) + for f in os.listdir(os.path.join(self.path, 'manifests')): + if f not in in_use_files: + try: + os.remove(os.path.join(self.path, 'manifests', f)) + except Exception as e: + self.log.warning(f'Failed to delete file "{f}": {e!r}') + def get_installed_game(self, app_name): if self._installed is None: try: @@ -221,3 +264,5 @@ class LGDLFS: with open(os.path.join(self.path, 'config.ini'), 'w') as cf: self.config.write(cf) + def get_dir_size(self): + return sum(f.stat().st_size for f in Path(self.path).glob('**/*') if f.is_file()) diff --git a/legendary/models/downloading.py b/legendary/models/downloading.py index 9ee9bad..b538090 100644 --- a/legendary/models/downloading.py +++ b/legendary/models/downloading.py @@ -28,7 +28,7 @@ class WriterTask: def __init__(self, filename, chunk_offset=0, chunk_size=0, chunk_guid=None, close=False, shared_memory=None, cache_file='', old_file='', release_memory=False, rename=False, - empty=False, kill=False, delete=False, old_filename='', fopen=False): + empty=False, kill=False, delete=False, old_filename='', fopen=False, silent=False): self.filename = filename self.empty = empty self.shm = shared_memory @@ -46,6 +46,7 @@ class WriterTask: self.rename = rename self.old_filename = old_filename + self.silent = silent # disable logging self.kill = kill # final task for worker (quit) @@ -96,7 +97,7 @@ class SharedMemorySegment: class ChunkTask: def __init__(self, chunk_guid, chunk_offset=0, chunk_size=0, cleanup=False, chunk_file=None): """ - Download amanger chunk task + Download manager chunk task :param chunk_guid: GUID of chunk :param cleanup: whether or not this chunk can be removed from disk/memory after it has been written @@ -113,7 +114,7 @@ class ChunkTask: class FileTask: def __init__(self, filename, delete=False, empty=False, fopen=False, close=False, - rename=False, temporary_filename=None): + rename=False, temporary_filename=None, silent=False): """ Download manager Task for a file @@ -130,6 +131,7 @@ class FileTask: self.close = close self.rename = rename self.temporary_filename = temporary_filename + self.silent = silent @property def is_reusing(self): diff --git a/legendary/models/egl.py b/legendary/models/egl.py index 7b6a6f5..c125cc1 100644 --- a/legendary/models/egl.py +++ b/legendary/models/egl.py @@ -58,6 +58,7 @@ class EGLManifest: self.display_name = None self.install_location = None self.install_size = None + self.install_tags = None self.installation_guid = None self.launch_command = None self.executable = None @@ -85,6 +86,7 @@ class EGLManifest: tmp.display_name = json.pop('DisplayName', '') tmp.install_location = json.pop('InstallLocation', '') tmp.install_size = json.pop('InstallSize', 0) + tmp.install_tags = json.pop('InstallTags', []) tmp.installation_guid = json.pop('InstallationGuid', '') tmp.launch_command = json.pop('LaunchCommand', '') tmp.executable = json.pop('LaunchExecutable', '') @@ -111,6 +113,7 @@ class EGLManifest: out['DisplayName'] = self.display_name out['InstallLocation'] = self.install_location out['InstallSize'] = self.install_size + out['InstallTags'] = self.install_tags out['InstallationGuid'] = self.installation_guid out['LaunchCommand'] = self.launch_command out['LaunchExecutable'] = self.executable @@ -136,6 +139,7 @@ class EGLManifest: tmp.display_name = igame.title tmp.install_location = igame.install_path tmp.install_size = igame.install_size + tmp.install_tags = igame.install_tags tmp.installation_guid = igame.egl_guid tmp.launch_command = igame.launch_parameters tmp.executable = igame.executable @@ -155,4 +159,4 @@ class EGLManifest: launch_parameters=self.launch_command, can_run_offline=self.can_run_offline, requires_ot=self.ownership_token, is_dlc=False, needs_verification=self.needs_validation, install_size=self.install_size, - egl_guid=self.installation_guid) + egl_guid=self.installation_guid, install_tags=self.install_tags) diff --git a/legendary/models/game.py b/legendary/models/game.py index 1e8c86c..16e2d79 100644 --- a/legendary/models/game.py +++ b/legendary/models/game.py @@ -79,7 +79,7 @@ class InstalledGame: def __init__(self, app_name='', title='', version='', manifest_path='', base_urls=None, install_path='', executable='', launch_parameters='', prereq_info=None, can_run_offline=False, requires_ot=False, is_dlc=False, save_path=None, - needs_verification=False, install_size=0, egl_guid=''): + needs_verification=False, install_size=0, egl_guid='', install_tags=None): self.app_name = app_name self.title = title self.version = version @@ -97,6 +97,7 @@ class InstalledGame: self.needs_verification = needs_verification self.install_size = install_size self.egl_guid = egl_guid + self.install_tags = install_tags if install_tags else [] @classmethod def from_json(cls, json): @@ -119,6 +120,7 @@ class InstalledGame: tmp.needs_verification = json.get('needs_verification', False) is True tmp.install_size = json.get('install_size', 0) tmp.egl_guid = json.get('egl_guid', '') + tmp.install_tags = json.get('install_tags', []) return tmp diff --git a/legendary/models/json_manifest.py b/legendary/models/json_manifest.py index 54ad2e3..e43ad75 100644 --- a/legendary/models/json_manifest.py +++ b/legendary/models/json_manifest.py @@ -157,15 +157,18 @@ class JSONFML(FML): _fm.chunk_parts = [] _fm.install_tags = _fmj.pop('InstallTags', list()) + _offset = 0 for _cpj in _fmj.pop('FileChunkParts'): _cp = ChunkPart() _cp.guid = guid_from_json(_cpj.pop('Guid')) _cp.offset = blob_to_num(_cpj.pop('Offset')) _cp.size = blob_to_num(_cpj.pop('Size')) + _cp.file_offset = _offset _fm.file_size += _cp.size if _cpj: print(f'Non-read ChunkPart keys: {_cpj.keys()}') _fm.chunk_parts.append(_cp) + _offset += _cp.size if _fmj: print(f'Non-read FileManifest keys: {_fmj.keys()}') diff --git a/legendary/models/manifest.py b/legendary/models/manifest.py index 2f88796..c66b9cb 100644 --- a/legendary/models/manifest.py +++ b/legendary/models/manifest.py @@ -507,13 +507,20 @@ class FML: # Each file is made up of "Chunk Parts" that can be spread across the "chunk stream" for fm in _fml.elements: _elem = struct.unpack(' 0: + logger.warning(f'Did not read {diff} bytes from chunk part!') + bio.seek(diff) # we have to calculate the actual file size ourselves for fm in _fml.elements: @@ -595,16 +602,17 @@ class FileManifest: return ''.format( - self.filename, self.symlink_target, self.hash.hex(), self.flags, - ', '.join(self.install_tags), cp_repr, self.file_size - ) + self.filename, self.symlink_target, self.hash.hex(), self.flags, + ', '.join(self.install_tags), cp_repr, self.file_size + ) class ChunkPart: - def __init__(self, guid=None, offset=0, size=0): + def __init__(self, guid=None, offset=0, size=0, file_offset=0): self.guid = guid self.offset = offset self.size = size + self.file_offset = file_offset # caches for things that are "expensive" to compute self._guid_str = None self._guid_num = None @@ -623,11 +631,11 @@ class ChunkPart: def __repr__(self): guid_readable = '-'.join('{:08x}'.format(g) for g in self.guid) - return ''.format( - guid_readable, self.offset, self.size) + return ''.format( + guid_readable, self.offset, self.size, self.file_offset) -class CustomFields: # this could probably be replaced with just a dict +class CustomFields: def __init__(self): self.size = 0 self.version = 0 @@ -644,6 +652,9 @@ class CustomFields: # this could probably be replaced with just a dict def __str__(self): return str(self._dict) + def items(self): + return self._dict.items() + def keys(self): return self._dict.keys() diff --git a/legendary/utils/cli.py b/legendary/utils/cli.py index c044144..2ee7fcd 100644 --- a/legendary/utils/cli.py +++ b/legendary/utils/cli.py @@ -1,3 +1,6 @@ +from legendary.utils.selective_dl import games + + def get_boolean_choice(prompt, default=True): if default: yn = 'Y/n' @@ -11,3 +14,31 @@ def get_boolean_choice(prompt, default=True): return True else: return False + + +def sdl_prompt(app_name, title): + tags = [''] + if '__required' in games[app_name]: + tags.extend(games[app_name]['__required']['tags']) + + print(f'You are about to install {title}, this game supports selective downloads.') + print('The following optional packs are available:') + for tag, info in games[app_name].items(): + if tag == '__required': + continue + print(' *', tag, '-', info['name']) + + print('Please enter a comma-separated list of optional packs to install (leave blank for defaults)') + examples = ','.join([g for g in games[app_name].keys() if g != '__required'][:2]) + choices = input(f'Additional packs [e.g. {examples}]: ') + if not choices: + return tags + + for c in choices.split(','): + c = c.strip() + if c in games[app_name]: + tags.extend(games[app_name][c]['tags']) + else: + print('Invalid tag:', c) + + return tags diff --git a/legendary/utils/game_workarounds.py b/legendary/utils/game_workarounds.py index 3da0d66..ecc8d60 100644 --- a/legendary/utils/game_workarounds.py +++ b/legendary/utils/game_workarounds.py @@ -1,10 +1,19 @@ # coding: utf-8 # games where the download order optimizations are enabled by default +# a set() of versions can be specified, empty set means all versions. _optimize_default = { - 'wombat', 'snapdragon' + 'wombat': {}, # world war z + 'snapdragon': {}, # metro exodus + 'honeycreeper': {}, # diabotical + 'bcc75c246fe04e45b0c1f1c3fd52503a': { # pillars of eternity + '1.0.2' # problematic version + } } -def is_opt_enabled(app_name): - return app_name.lower() in _optimize_default +def is_opt_enabled(app_name, version): + if (versions := _optimize_default.get(app_name.lower())) is not None: + if version in versions or not versions: + return True + return False diff --git a/legendary/utils/lfs.py b/legendary/utils/lfs.py index c549cff..f9accac 100644 --- a/legendary/utils/lfs.py +++ b/legendary/utils/lfs.py @@ -27,7 +27,8 @@ def delete_folder(path: str, recursive=True) -> bool: def delete_filelist(path: str, filenames: List[str], - delete_root_directory: bool = False) -> bool: + delete_root_directory: bool = False, + silent: bool = False) -> bool: dirs = set() no_error = True @@ -40,7 +41,8 @@ def delete_filelist(path: str, filenames: List[str], try: os.remove(os.path.join(path, _dir, _fn)) except Exception as e: - logger.error(f'Failed deleting file {filename} with {e!r}') + if not silent: + logger.error(f'Failed deleting file {filename} with {e!r}') no_error = False # add intermediate directories that would have been missed otherwise @@ -58,14 +60,16 @@ def delete_filelist(path: str, filenames: List[str], # directory has already been deleted, ignore that continue except Exception as e: - logger.error(f'Failed removing directory "{_dir}" with {e!r}') + if not silent: + logger.error(f'Failed removing directory "{_dir}" with {e!r}') no_error = False if delete_root_directory: try: os.rmdir(path) except Exception as e: - logger.error(f'Removing game directory failed with {e!r}') + if not silent: + logger.error(f'Removing game directory failed with {e!r}') return no_error diff --git a/legendary/utils/manifests.py b/legendary/utils/manifests.py new file mode 100644 index 0000000..aa85d44 --- /dev/null +++ b/legendary/utils/manifests.py @@ -0,0 +1,28 @@ +from legendary.models.manifest import Manifest + + +def combine_manifests(base_manifest: Manifest, delta_manifest: Manifest): + added = set() + # overwrite file elements with the ones from the delta manifest + for idx, file_elem in enumerate(base_manifest.file_manifest_list.elements): + try: + delta_file = delta_manifest.file_manifest_list.get_file_by_path(file_elem.filename) + base_manifest.file_manifest_list.elements[idx] = delta_file + added.add(delta_file.filename) + except ValueError: + pass + + # add other files that may be missing + for delta_file in delta_manifest.file_manifest_list.elements: + if delta_file.filename not in added: + base_manifest.file_manifest_list.elements.append(delta_file) + # update count and clear map + base_manifest.file_manifest_list.count = len(base_manifest.file_manifest_list.elements) + base_manifest.file_manifest_list._path_map = None + + # add chunks from delta manifest to main manifest and again clear path caches + base_manifest.chunk_data_list.elements.extend(delta_manifest.chunk_data_list.elements) + base_manifest.chunk_data_list.count = len(base_manifest.chunk_data_list.elements) + base_manifest.chunk_data_list._guid_map = None + base_manifest.chunk_data_list._guid_int_map = None + base_manifest.chunk_data_list._path_map = None diff --git a/legendary/utils/savegame_helper.py b/legendary/utils/savegame_helper.py index 48f4a8f..d7b36e8 100644 --- a/legendary/utils/savegame_helper.py +++ b/legendary/utils/savegame_helper.py @@ -123,7 +123,8 @@ class SaveGameHelper: # create chunk part and write it to chunk buffer cp = ChunkPart(guid=cur_chunk.guid, offset=cur_buffer.tell(), - size=min(remaining, 1024 * 1024 - cur_buffer.tell())) + size=min(remaining, 1024 * 1024 - cur_buffer.tell()), + file_offset=cf.tell()) _tmp = cf.read(cp.size) if not _tmp: self.log.warning(f'Got EOF for "{f.filename}" with {remaining} bytes remaining! ' diff --git a/legendary/utils/selective_dl.py b/legendary/utils/selective_dl.py new file mode 100644 index 0000000..c0ec31a --- /dev/null +++ b/legendary/utils/selective_dl.py @@ -0,0 +1,38 @@ +# This file contains definitions for selective downloading for supported games +# coding: utf-8 + +_cyberpunk_sdl = { + 'de': {'tags': ['voice_de_de'], 'name': 'Deutsch'}, + 'es': {'tags': ['voice_es_es'], 'name': 'español (España)'}, + 'fr': {'tags': ['voice_fr_fr'], 'name': 'français'}, + 'it': {'tags': ['voice_it_it'], 'name': 'italiano'}, + 'ja': {'tags': ['voice_ja_jp'], 'name': '日本語'}, + 'ko': {'tags': ['voice_ko_kr'], 'name': '한국어'}, + 'pl': {'tags': ['voice_pl_pl'], 'name': 'polski'}, + 'pt': {'tags': ['voice_pt_br'], 'name': 'português brasileiro'}, + 'ru': {'tags': ['voice_ru_ru'], 'name': 'русский'}, + 'cn': {'tags': ['voice_zh_cn'], 'name': '中文（中国）'} +} + +_fortnite_sdl = { + '__required': {'tags': ['chunk0', 'chunk10'], 'name': 'Fortnite Core'}, + 'stw': {'tags': ['chunk11', 'chunk11optional'], 'name': 'Fortnite Save the World'}, + 'hd_textures': {'tags': ['chunk10optional'], 'name': 'High Resolution Textures'}, + 'lang_de': {'tags': ['chunk2'], 'name': '(Language Pack) Deutsch'}, + 'lang_fr': {'tags': ['chunk5'], 'name': '(Language Pack) français'}, + 'lang_pl': {'tags': ['chunk7'], 'name': '(Language Pack) polski'}, + 'lang_ru': {'tags': ['chunk8'], 'name': '(Language Pack) русский'}, + 'lang_cn': {'tags': ['chunk9'], 'name': '(Language Pack) 中文（中国）'} +} + +games = { + 'Fortnite': _fortnite_sdl, + 'Ginger': _cyberpunk_sdl +} + + +def get_sdl_appname(app_name): + for k in games.keys(): + if app_name.startswith(k): + return k + return None diff --git a/legendary/utils/wine_helpers.py b/legendary/utils/wine_helpers.py new file mode 100644 index 0000000..23c54ab --- /dev/null +++ b/legendary/utils/wine_helpers.py @@ -0,0 +1,17 @@ +import os +import configparser + + +def read_registry(wine_pfx): + reg = configparser.ConfigParser(comment_prefixes=(';', '#', '/', 'WINE'), allow_no_value=True) + reg.optionxform = str + reg.read(os.path.join(wine_pfx, 'user.reg')) + return reg + + +def get_shell_folders(registry, wine_pfx): + folders = dict() + for k, v in registry['Software\\\\Microsoft\\\\Windows\\\\CurrentVersion\\\\Explorer\\\\Shell Folders'].items(): + path_cleaned = v.strip('"').strip().replace('\\\\', '/').replace('C:/', '') + folders[k.strip('"').strip()] = os.path.join(wine_pfx, 'drive_c', path_cleaned) + return folders