Merge 05be598e0a into 42af7b5db7

README.md

@@ -746,8 +746,6 @@ wrapper = "/path/with spaces/gamemoderun"
 no_wine = true
 ; Override the executable launched for this game, for example to bypass a launcher (e.g. Borderlands)
 override_exe = relative/path/to/file.exe
-; Disable selective downloading for this title
-disable_sdl = true
 
 [AppName3]
 ; Command to run before launching the gmae

legendary/cli.py

@@ -16,6 +16,7 @@ from logging.handlers import QueueListener
 from multiprocessing import freeze_support, Queue as MPQueue
 from platform import platform
 from sys import exit, stdout, platform as sys_platform
+from epic_expreval import Tokenizer
 
 from legendary import __version__, __codename__
 from legendary.core import LegendaryCore
@@ -27,7 +28,7 @@ from legendary.utils.custom_parser import HiddenAliasSubparsersAction
 from legendary.utils.env import is_windows_mac_or_pyi
 from legendary.lfs.eos import add_registry_entries, query_registry_entries, remove_registry_entries
 from legendary.lfs.utils import validate_files, clean_filename
-from legendary.utils.selective_dl import get_sdl_appname
+from legendary.utils.selective_dl import get_sdl_data, LGDEvaluationContext, parse_components_selection, EXTRA_FUNCTIONS
 from legendary.lfs.wine_helpers import read_registry, get_shell_folders, case_insensitive_file_search
 
 # todo custom formatter for cli logger (clean info, highlighted error/warning)
@@ -911,52 +912,30 @@ class LegendaryCLI:
             else:
                 logger.info(f'Using existing repair file: {repair_file}')
 
-        # check if SDL should be disabled
-        sdl_enabled = not args.install_tag and not game.is_dlc
-        config_tags = self.core.lgd.config.get(game.app_name, 'install_tags', fallback=None)
-        config_disable_sdl = self.core.lgd.config.getboolean(game.app_name, 'disable_sdl', fallback=False)
-        # remove config flag if SDL is reset
-        if config_disable_sdl and args.reset_sdl and not args.disable_sdl:
-            self.core.lgd.config.remove_option(game.app_name, 'disable_sdl')
-        # if config flag is not yet set, set it and remove previous install tags
-        elif not config_disable_sdl and args.disable_sdl:
-            logger.info('Clearing install tags from config and disabling SDL for title.')
-            if config_tags:
-                self.core.lgd.config.remove_option(game.app_name, 'install_tags')
-                config_tags = None
-            self.core.lgd.config.set(game.app_name, 'disable_sdl', 'true')
-            sdl_enabled = False
-        # just disable SDL, but keep config tags that have been manually specified
-        elif config_disable_sdl or args.disable_sdl:
-            sdl_enabled = False
-
-        if sdl_enabled and ((sdl_name := get_sdl_appname(game.app_name)) is not None):
-            if not self.core.is_installed(game.app_name) or config_tags is None or args.reset_sdl:
-                sdl_data = self.core.get_sdl_data(sdl_name, platform=args.platform)
-                if sdl_data:
-                    if args.skip_sdl:
-                        args.install_tag = ['']
-                        if '__required' in sdl_data:
-                            args.install_tag.extend(sdl_data['__required']['tags'])
-                    else:
-                        args.install_tag = sdl_prompt(sdl_data, game.app_title)
-                    self.core.lgd.config.set(game.app_name, 'install_tags', ','.join(args.install_tag))
-                else:
-                    logger.error(f'Unable to get SDL data for {sdl_name}')
+        logger.info('Checking for install components')
+        config_options = self.core.lgd.config.get(game.app_name, 'install_components', fallback=None)
+        install_components = args.install_component or []
+        sdl_data = get_sdl_data(self.core.lgd.egl_content_path, game.app_name, game.app_version(args.platform))
+        context = LGDEvaluationContext(self.core)
+        if not self.core.is_installed(game.app_name) or config_options is None or args.reset_sdl:
+            if sdl_data:
+                if not args.skip_sdl:
+                    install_components = sdl_prompt(sdl_data, game.app_title, context)
             else:
-                args.install_tag = config_tags.split(',')
-        elif args.install_tag and not game.is_dlc and not args.no_install:
-            config_tags = ','.join(args.install_tag)
-            logger.info(f'Saving install tags for "{game.app_name}" to config: {config_tags}')
-            self.core.lgd.config.set(game.app_name, 'install_tags', config_tags)
-        elif not game.is_dlc:
-            if config_tags and args.reset_sdl:
-                logger.info('Clearing install tags from config.')
-                self.core.lgd.config.remove_option(game.app_name, 'install_tags')
-            elif config_tags:
-                logger.info(f'Using install tags from config: {config_tags}')
-                args.install_tag = config_tags.split(',')
+                logger.error(f'Unable to get SDL data for {game.app_name}')
+        else:
+            install_components = config_options.split(',')
+        context.selection = set(install_components)
+            
+        install_tags = set()
+        if sdl_data:
+            parse_components_selection(sdl_data, context, install_components, install_tags)
 
+        if install_components:
+            self.core.lgd.config.set(game.app_name, 'install_components', ','.join(install_components))
+        install_tags = list(install_tags)
+        logger.debug(f'Selected components: {install_components}')
+        logger.debug(f'Selected tags: {install_tags}')
         logger.info(f'Preparing download for "{game.app_title}" ({game.app_name})...')
         # todo use status queue to print progress from CLI
         # This has become a little ridiculous hasn't it?
@@ -969,8 +948,10 @@ class LegendaryCLI:
                                                           override_base_url=args.override_base_url,
                                                           platform=args.platform,
                                                           file_prefix_filter=args.file_prefix,
+                                                          file_suffix_filter=args.file_suffix,
                                                           file_exclude_filter=args.file_exclude_prefix,
-                                                          file_install_tag=args.install_tag,
+                                                          file_install_tag=install_tags,
+                                                          game_install_components=install_components,
                                                           dl_optimizations=args.order_opt,
                                                           dl_timeout=args.dl_timeout,
                                                           repair=args.repair_mode,
@@ -1642,6 +1623,7 @@ class LegendaryCLI:
                            f'not being available on the selected platform or currently logged-in account.')
             args.offline = True
 
+        sdl_data = get_sdl_data(self.core.lgd.egl_content_path, app_name, game.app_version(args.platform)) or {}
         manifest_data = None
         entitlements = None
         # load installed manifest or URI
@@ -1761,11 +1743,11 @@ class LegendaryCLI:
                                                   igame.save_path))
                 installation_info.append(InfoItem('EGL sync GUID', 'synced_egl_guid', igame.egl_guid,
                                                   igame.egl_guid))
-                if igame.install_tags:
-                    tags = ', '.join(igame.install_tags)
+                if igame.install_components:
+                    opts = ', '.join(igame.install_components)
                 else:
-                    tags = '(None, all game data selected for install)'
-                installation_info.append(InfoItem('Install tags', 'install_tags', tags, igame.install_tags))
+                    opts = '(None, all game data selected for install)'
+                installation_info.append(InfoItem('Install components', 'install_components', opts, igame.install_components))
                 installation_info.append(InfoItem('Requires ownership verification token (DRM)', 'requires_ovt',
                                                   igame.requires_ot, igame.requires_ot))
 
@@ -1834,14 +1816,51 @@ class LegendaryCLI:
             else:
                 manifest_info.append(InfoItem('Uninstaller', 'uninstaller', None, None))
 
-            install_tags = {''}
-            for fm in manifest.file_manifest_list.elements:
-                for tag in fm.install_tags:
-                    install_tags.add(tag)
+            if sdl_data:
+                context = LGDEvaluationContext(self.core)
+                install_tags_human = []
+                install_tags_data = []
+                for element in sdl_data['Data']:
+                    if not element.get('Title'):
+                        continue
 
-            install_tags = sorted(install_tags)
-            install_tags_human = ', '.join(i if i else '(empty)' for i in install_tags)
-            manifest_info.append(InfoItem('Install tags', 'install_tags', install_tags_human, install_tags))
+                    is_required = element.get('IsRequired','false')=='true'
+                    is_default_selected = element.get('IsDefaultSelected','false')=='true'
+                    if is_default_selected and element.get('DefaultSelectedExpression'):
+                        tk = Tokenizer(element['DefaultSelectedExpression'], context)
+                        tk.extend_functions(EXTRA_FUNCTIONS)
+                        tk.compile()
+                        is_default_selected = tk.execute('')
+                    # This mapping abstracts expressions away from info command
+                    # marking default for options that apply to given user
+                    mapped_element = {
+                        'UniqueId': element.get('UniqueId'),
+                        'IsRequired': is_required,
+                        'IsDefaultSelected': is_default_selected
+                    }
+                    install_tags_data.append(mapped_element)
+
+                    for key in element.keys():
+                        if key.endswith('_translate'):
+                            mapped_element[key] = element[key] == 'true'
+                        elif key.startswith('Title') or key.startswith('Description'):
+                            mapped_element[key] = element[key]
+
+                    required_txt = ' (required)' if is_required else ''
+                    if element.get('Children'):
+                        mapped_element['ConfigHandler'] = element['ConfigHandler']
+                        mapped_element['Children'] = element['Children']
+
+                        install_tags_human.append(f'{element["Title"]}{required_txt}')
+                        for child in element.get('Children', []):
+                            install_tags_human.append(f'\t{child["UniqueId"]} - {child["Title"]}')
+                    else:
+                        install_tags_human.append(f'{element["UniqueId"]} - {element["Title"]}{required_txt}')
+            else:
+                install_tags_human = '(none)'
+                install_tags_data = None
+            
+            manifest_info.append(InfoItem('Install components', 'install_components', install_tags_human, install_tags_data))
             # file and chunk count
             manifest_info.append(InfoItem('Files', 'num_files', manifest.file_manifest_list.count,
                                           manifest.file_manifest_list.count))
@@ -1857,42 +1876,6 @@ class LegendaryCLI:
             manifest_info.append(InfoItem('Download size (compressed)', 'download_size',
                                           chunk_size, total_size))
 
-            # if there are install tags break down size by tag
-            tag_disk_size = []
-            tag_disk_size_human = []
-            tag_download_size = []
-            tag_download_size_human = []
-            if len(install_tags) > 1:
-                longest_tag = max(max(len(t) for t in install_tags), len('(empty)'))
-                for tag in install_tags:
-                    # sum up all file sizes for the tag
-                    human_tag = tag or '(empty)'
-                    tag_files = [fm for fm in manifest.file_manifest_list.elements if
-                                 (tag in fm.install_tags) or (not tag and not fm.install_tags)]
-                    tag_file_size = sum(fm.file_size for fm in tag_files)
-                    tag_disk_size.append(dict(tag=tag, size=tag_file_size, count=len(tag_files)))
-                    tag_file_size_human = '{:.02f} GiB'.format(tag_file_size / 1024 / 1024 / 1024)
-                    tag_disk_size_human.append(f'{human_tag.ljust(longest_tag)} - {tag_file_size_human} '
-                                               f'(Files: {len(tag_files)})')
-                    # tag_disk_size_human.append(f'Size: {tag_file_size_human}, Files: {len(tag_files)}, Tag: "{tag}"')
-                    # accumulate chunk guids used for this tag and count their size too
-                    tag_chunk_guids = set()
-                    for fm in tag_files:
-                        for cp in fm.chunk_parts:
-                            tag_chunk_guids.add(cp.guid_num)
-
-                    tag_chunk_size = sum(c.file_size for c in manifest.chunk_data_list.elements
-                                         if c.guid_num in tag_chunk_guids)
-                    tag_download_size.append(dict(tag=tag, size=tag_chunk_size, count=len(tag_chunk_guids)))
-                    tag_chunk_size_human = '{:.02f} GiB'.format(tag_chunk_size / 1024 / 1024 / 1024)
-                    tag_download_size_human.append(f'{human_tag.ljust(longest_tag)} - {tag_chunk_size_human} '
-                                                   f'(Chunks: {len(tag_chunk_guids)})')
-
-            manifest_info.append(InfoItem('Disk size by install tag', 'tag_disk_size',
-                                          tag_disk_size_human or 'N/A', tag_disk_size))
-            manifest_info.append(InfoItem('Download size by install tag', 'tag_download_size',
-                                          tag_download_size_human or 'N/A', tag_download_size))
-
         if not args.json:
             def print_info_item(item: InfoItem):
                 if item.value is None:
@@ -2625,7 +2608,49 @@ class LegendaryCLI:
         igame.install_path = new_path
         self.core.install_game(igame)
         logger.info('Finished.')
+    
+    def get_install_size(self, args):
+        args.app_name = self._resolve_aliases(args.app_name)
 
+        game = self.core.get_game(args.app_name, update_meta=False, platform=args.platform)
+        if not game:
+            game = self.core.get_game(args.app_name, update_meta=True, platform=args.platform)
+
+        version = game.app_version(args.platform)
+        manifest_data = self.core.lgd.load_manifest(game.app_name, version, args.platform)
+        if not manifest_data:
+            manifest_data, _ = self.core.get_cdn_manifest(game, platform=args.platform)
+            self.core.lgd.save_manifest(game.app_name, manifest_data, version, args.platform)
+
+        manifest = self.core.load_manifest(manifest_data)
+        sdl_data = get_sdl_data(self.core.lgd.egl_content_path, game.app_name, version) or {}
+        install_components = args.install_component or []
+        install_tags = set()
+        if sdl_data:
+            context = LGDEvaluationContext(self.core)
+            context.selection.update(install_components)
+            parse_components_selection(sdl_data, context, install_components, install_tags)
+
+        files = manifest.file_manifest_list.elements
+        filtered = [f for f in files if any(tag in install_tags for tag in f.install_tags)]
+        
+        calculated_chunks = set()
+        install_size = 0
+        download_size = 0
+        for file in filtered:
+            install_size += file.file_size
+            for chunk in file.chunk_parts:
+                if chunk.guid_num in calculated_chunks:
+                    continue
+                data = manifest.chunk_data_list.get_chunk_by_guid_num(chunk.guid_num)
+                download_size += data.file_size
+                calculated_chunks.add(chunk.guid_num)
+
+        if args.json:
+            print(json.dumps({'download': download_size, 'install': install_size}))
+        else:
+            print(f'- Download size: {download_size / 1024 / 1024 / 1024:.02f} GiB')
+            print(f'- Install size: {install_size / 1024 / 1024 / 1024:.02f} GiB')
 
 def main():
     # Set output encoding to UTF-8 if not outputting to a terminal
@@ -2683,6 +2708,8 @@ def main():
 
     # hidden commands have no help text
     get_token_parser = subparsers.add_parser('get-token')
+    install_size_parser = subparsers.add_parser('install-size')
+
 
     # Positional arguments
     install_parser.add_argument('app_name', help='Name of the app', metavar='<App Name>')
@@ -2764,10 +2791,12 @@ def main():
                                 help='Platform for install (default: installed or Windows)')
     install_parser.add_argument('--prefix', dest='file_prefix', action='append', metavar='<prefix>',
                                 help='Only fetch files whose path starts with <prefix> (case insensitive)')
+    install_parser.add_argument('--suffix', dest='file_suffix', action='append', metavar='<suffix>',
+                                help='Only fetch files whose path ends with <prefix> (case insensitive)')
     install_parser.add_argument('--exclude', dest='file_exclude_prefix', action='append', metavar='<prefix>',
                                 type=str, help='Exclude files starting with <prefix> (case insensitive)')
-    install_parser.add_argument('--install-tag', dest='install_tag', action='append', metavar='<tag>',
-                                type=str, help='Only download files with the specified install tag')
+    install_parser.add_argument('--install-component', dest='install_component', action='append', metavar='<id>',
+                                type=str, help='Only download files with the specified optional download id')
     install_parser.add_argument('--enable-reordering', dest='order_opt', action='store_true',
                                 help='Enable reordering optimization to reduce RAM requirements '
                                      'during download (may have adverse results for some titles)')
@@ -2787,8 +2816,6 @@ def main():
                                 help='Reset selective downloading choices (requires repair to download new components)')
     install_parser.add_argument('--skip-sdl', dest='skip_sdl', action='store_true',
                                 help='Skip SDL prompt and continue with defaults (only required game data)')
-    install_parser.add_argument('--disable-sdl', dest='disable_sdl', action='store_true',
-                                help='Disable selective downloading for title, reset existing configuration (if any)')
     install_parser.add_argument('--preferred-cdn', dest='preferred_cdn', action='store', metavar='<hostname>',
                                 help='Set the hostname of the preferred CDN to use when available')
     install_parser.add_argument('--no-https', dest='disable_https', action='store_true',
@@ -2800,6 +2827,16 @@ def main():
     install_parser.add_argument('--bind', dest='bind_ip', action='store', metavar='<IPs>', type=str,
                                 help='Comma-separated list of IPs to bind to for downloading')
 
+    install_size_parser.add_argument('app_name', metavar='<App Name>')
+    install_size_parser.add_argument('--install-component', dest='install_component', action='append', metavar='<id>',
+                                type=str, help='Specify what component should be treated as selected')
+    
+    install_size_parser.add_argument('--platform', dest='platform', action='store', metavar='<Platform>', type=str,
+                                help='Platform for install (default: Windows)')
+    
+    install_size_parser.add_argument('--json', dest='json', action='store_true',
+                               help='Print information as JSON')
+
     uninstall_parser.add_argument('--keep-files', dest='keep_files', action='store_true',
                                   help='Keep files but remove game from Legendary database')
     uninstall_parser.add_argument('--skip-uninstaller', dest='skip_uninstaller', action='store_true',
@@ -2896,8 +2933,8 @@ def main():
     list_files_parser.add_argument('--json', dest='json', action='store_true', help='Output in JSON format')
     list_files_parser.add_argument('--hashlist', dest='hashlist', action='store_true',
                                    help='Output file hash list in hashcheck/sha1sum -c compatible format')
-    list_files_parser.add_argument('--install-tag', dest='install_tag', action='store', metavar='<tag>',
-                                   type=str, help='Show only files with specified install tag')
+    list_files_parser.add_argument('--install-component', dest='install_component', action='store', metavar='<id>',
+                                   type=str, help='Show only files with specified optional download id')
 
     sync_saves_parser.add_argument('--skip-upload', dest='download_only', action='store_true',
                                    help='Only download new saves from cloud, don\'t upload')
@@ -3113,6 +3150,8 @@ def main():
             cli.crossover_setup(args)
         elif args.subparser_name == 'move':
             cli.move(args)
+        elif args.subparser_name == 'install-size':
+            cli.get_install_size(args)
     except KeyboardInterrupt:
         logger.info('Command was aborted via KeyboardInterrupt, cleaning up...')
 

legendary/core.py

@@ -38,7 +38,6 @@ from legendary.utils.env import is_windows_mac_or_pyi
 from legendary.lfs.eos import EOSOverlayApp, query_registry_entries
 from legendary.utils.game_workarounds import is_opt_enabled, update_workarounds, get_exe_override
 from legendary.utils.savegame_helper import SaveGameHelper
-from legendary.utils.selective_dl import games as sdl_games
 from legendary.lfs.wine_helpers import read_registry, get_shell_folders, case_insensitive_path_search
 
 
@@ -226,6 +225,7 @@ class LegendaryCore:
                 try:
                     self.egs.resume_session(lock.data)
                     self.logged_in = True
+                    self.update_launcher_content()
                     return True
                 except InvalidCredentialsError as e:
                     self.log.warning(f'Resuming failed due to invalid credentials: {e!r}')
@@ -247,6 +247,7 @@ class LegendaryCore:
 
         lock.data = userdata
         self.logged_in = True
+        self.update_launcher_content()
         return True
 
     def login(self, force_refresh=False) -> bool:
@@ -285,22 +286,56 @@ class LegendaryCore:
             self.log.debug('No cached legendary config to apply.')
             return
 
-        if 'egl_config' in version_info:
-            self.egs.update_egs_params(version_info['egl_config'])
-            self._egl_version = version_info['egl_config'].get('version', self._egl_version)
-            for data_key in version_info['egl_config'].get('data_keys', []):
+        if egl_config := version_info.get('egl_config'):
+            self.egs.update_egs_params(egl_config)
+            self._egl_version = egl_config.get('version', self._egl_version)
+            for data_key in egl_config.get('data_keys', []):
                 if data_key not in self.egl.data_keys:
                     self.egl.data_keys.append(data_key)
         if game_overrides := version_info.get('game_overrides'):
             update_workarounds(game_overrides)
-            if sdl_config := game_overrides.get('sdl_config'):
-                # add placeholder for games to fetch from API that aren't hardcoded
-                for app_name in sdl_config.keys():
-                    if app_name not in sdl_games:
-                        sdl_games[app_name] = None
         if lgd_config := version_info.get('legendary_config'):
             self.webview_killswitch = lgd_config.get('webview_killswitch', False)
 
+    def update_launcher_content(self) -> None:
+        """Updates metadata we need from the EGL manifest"""
+        cached = self.lgd.get_cached_egl_content_version()
+        if cached['version'] and (datetime.now().timestamp() - cached['last_update']) < 24*3600:
+            self.log.debug('EGL content version cache is still fresh, not updating')
+            return
+
+        assets = self.egs.get_launcher_manifests()
+        asset = next((
+            asset for asset in assets['elements']
+            if asset['appName'] == 'EpicGamesLauncherContent'
+        ), None)
+
+        if not asset:
+            raise ValueError('"EpicGamesLauncherContent" asset not found in launcher manifests')
+
+        if asset['buildVersion'] == cached['version']:
+            self.log.debug('EGL content is up-to-date')
+            self.lgd.set_cached_egl_content_version(asset['buildVersion'])
+            return
+
+        manifest_urls, base_urls, manifest_hash = self._get_cdn_urls_for_asset(asset)
+        manifest_bytes = self._download_manifest(manifest_urls, manifest_hash)
+        manifest = self.load_manifest(manifest_bytes)
+
+        dlm = DLManager(self.lgd.egl_content_path, base_urls[0])
+        dlm.log.setLevel(logging.WARNING)
+
+        suffixes = [
+            # We need sdmeta files for selective downloads
+            'sdmeta'
+        ]
+
+        dlm.run_analysis(manifest=manifest, file_suffix_filter=suffixes)
+        dlm.start()
+        dlm.join()
+        self.log.info(f'EGL content updated from {cached["version"]} to {asset["buildVersion"]}')
+        self.lgd.set_cached_egl_content_version(asset['buildVersion'])
+
     def get_egl_version(self):
         return self._egl_version
 
@@ -314,31 +349,6 @@ class LegendaryCore:
 
         return update_info.get('game_wiki', {}).get(app_name, {}).get(sys_platform)
 
-    def get_sdl_data(self, app_name, platform='Windows'):
-        if platform not in ('Win32', 'Windows'):
-            app_name = f'{app_name}_{platform}'
-
-        if app_name not in sdl_games:
-            return None
-        # load hardcoded data as fallback
-        sdl_data = sdl_games[app_name]
-        # get cached data
-        cached = self.lgd.get_cached_sdl_data(app_name)
-        # check if newer version is available and/or download if necessary
-        version_info = self.lgd.get_cached_version()['data']
-        latest = version_info.get('game_overrides', {}).get('sdl_config', {}).get(app_name)
-        if (not cached and latest) or (cached and latest and latest > cached['version']):
-            try:
-                sdl_data = self.lgdapi.get_sdl_config(app_name)
-                self.log.debug(f'Downloaded SDL data for "{app_name}", version: {latest}')
-                self.lgd.set_cached_sdl_data(app_name, latest, sdl_data)
-            except Exception as e:
-                self.log.warning(f'Downloading SDL data failed with {e!r}')
-        elif cached:
-            sdl_data = cached['data']
-        # return data if available
-        return sdl_data
-
     def update_aliases(self, force=False):
         _aliases_enabled = not self.lgd.config.getboolean('Legendary', 'disable_auto_aliasing', fallback=False)
         if _aliases_enabled and (force or not self.lgd.aliases):
@@ -1252,10 +1262,14 @@ class LegendaryCore:
         if len(m_api_r['elements']) > 1:
             raise ValueError('Manifest response has more than one element!')
 
-        manifest_hash = m_api_r['elements'][0]['hash']
+        return self._get_cdn_urls_for_asset(m_api_r['elements'][0])
+
+    @staticmethod
+    def _get_cdn_urls_for_asset(asset):
+        manifest_hash = asset['hash']
         base_urls = []
         manifest_urls = []
-        for manifest in m_api_r['elements'][0]['manifests']:
+        for manifest in asset['manifests']:
             base_url = manifest['uri'].rpartition('/')[0]
             if base_url not in base_urls:
                 base_urls.append(base_url)
@@ -1276,6 +1290,9 @@ class LegendaryCore:
         if disable_https:
             manifest_urls = [url.replace('https://', 'http://') for url in manifest_urls]
 
+        return self._download_manifest(manifest_urls, manifest_hash), base_urls
+
+    def _download_manifest(self, manifest_urls: list[str], manifest_hash: str):
         for url in manifest_urls:
             self.log.debug(f'Trying to download manifest from "{url}"...')
             try:
@@ -1297,7 +1314,7 @@ class LegendaryCore:
         if sha1(manifest_bytes).hexdigest() != manifest_hash:
             raise ValueError('Manifest sha hash mismatch!')
 
-        return manifest_bytes, base_urls
+        return manifest_bytes
 
     def get_uri_manifest(self, uri):
         if uri.startswith('http'):
@@ -1326,12 +1343,13 @@ class LegendaryCore:
                          game_folder: str = '', override_manifest: str = '',
                          override_old_manifest: str = '', override_base_url: str = '',
                          platform: str = 'Windows', file_prefix_filter: list = None,
-                         file_exclude_filter: list = None, file_install_tag: list = None,
-                         dl_optimizations: bool = False, dl_timeout: int = 10,
-                         repair: bool = False, repair_use_latest: bool = False,
-                         disable_delta: bool = False, override_delta_manifest: str = '',
-                         egl_guid: str = '', preferred_cdn: str = None,
-                         disable_https: bool = False, bind_ip: str = None) -> (DLManager, AnalysisResult, ManifestMeta):
+                         file_suffix_filter: list = None, file_exclude_filter: list = None,
+                         file_install_tag: list = None, dl_optimizations: bool = False,
+                         dl_timeout: int = 10, repair: bool = False,
+                         repair_use_latest: bool = False, disable_delta: bool = False,
+                         override_delta_manifest: str = '', egl_guid: str = '',
+                         preferred_cdn: str = None, disable_https: bool = False,
+                         bind_ip: str = None, game_install_components: list = None) -> (DLManager, AnalysisResult, ManifestMeta):
         # load old manifest
         old_manifest = None
 
@@ -1502,6 +1520,7 @@ class LegendaryCore:
         anlres = dlm.run_analysis(manifest=new_manifest, old_manifest=old_manifest,
                                   patch=not disable_patching, resume=not force,
                                   file_prefix_filter=file_prefix_filter,
+                                  file_suffix_filter=file_suffix_filter,
                                   file_exclude_filter=file_exclude_filter,
                                   file_install_tag=file_install_tag,
                                   processing_optimization=process_opt)
@@ -1539,6 +1558,7 @@ class LegendaryCore:
                               can_run_offline=offline == 'true', requires_ot=ot == 'true',
                               is_dlc=base_game is not None, install_size=anlres.install_size,
                               egl_guid=egl_guid, install_tags=file_install_tag,
+                              install_components=game_install_components,
                               platform=platform, uninstaller=uninstaller)
 
         return dlm, anlres, igame
@@ -1852,9 +1872,10 @@ class LegendaryCore:
                                version=new_manifest.meta.build_version,
                                platform='Windows')
 
+        # TODO: Migrate this to components instead?
         # transfer install tag choices to config
-        if lgd_igame.install_tags:
-            self.lgd.config.set(app_name, 'install_tags', ','.join(lgd_igame.install_tags))
+        #if lgd_igame.install_tags:
+        #    self.lgd.config.set(app_name, 'install_tags', ','.join(lgd_igame.install_tags))
 
         # mark game as installed
         _ = self._install_game(lgd_igame)

legendary/downloader/mp/manager.py

@@ -81,8 +81,8 @@ class DLManager(Process):
 
     def run_analysis(self, manifest: Manifest, old_manifest: Manifest = None,
                      patch=True, resume=True, file_prefix_filter=None,
-                     file_exclude_filter=None, file_install_tag=None,
-                     processing_optimization=False) -> AnalysisResult:
+                     file_suffix_filter=None, file_exclude_filter=None,
+                     file_install_tag=None, processing_optimization=False) -> AnalysisResult:
         """
         Run analysis on manifest and old manifest (if not None) and return a result
         with a summary resources required in order to install the provided manifest.
@@ -92,6 +92,7 @@ class DLManager(Process):
         :param patch: Patch instead of redownloading the entire file
         :param resume: Continue based on resume file if it exists
         :param file_prefix_filter: Only download files that start with this prefix
+        :param file_suffix_filter: Only download files that end with this prefix
         :param file_exclude_filter: Exclude files with this prefix from download
         :param file_install_tag: Only install files with the specified tag
         :param processing_optimization: Attempt to optimize processing order and RAM usage
@@ -187,19 +188,24 @@ class DLManager(Process):
             mc.changed -= files_to_skip
             mc.unchanged |= files_to_skip
 
-        if file_prefix_filter:
-            if isinstance(file_prefix_filter, str):
-                file_prefix_filter = [file_prefix_filter]
+        if file_prefix_filter or file_suffix_filter:
+            file_prefix_filter = file_prefix_filter or []
+            file_suffix_filter = file_suffix_filter or []
 
             file_prefix_filter = [f.lower() for f in file_prefix_filter]
-            files_to_skip = set(i.filename for i in manifest.file_manifest_list.elements if not
-                                any(i.filename.lower().startswith(pfx) for pfx in file_prefix_filter))
+            file_suffix_filter = [f.lower() for f in file_suffix_filter]
+            files_to_skip = set(
+                i.filename
+                for i in manifest.file_manifest_list.elements
+                if not any(i.filename.lower().startswith(pfx) for pfx in file_prefix_filter)
+                and not any(i.filename.lower().endswith(sfx) for sfx in file_suffix_filter)
+            )
             self.log.info(f'Found {len(files_to_skip)} files to skip based on include prefix(es)')
             mc.added -= files_to_skip
             mc.changed -= files_to_skip
             mc.unchanged |= files_to_skip
 
-        if file_prefix_filter or file_exclude_filter or file_install_tag:
+        if file_prefix_filter or file_suffix_filter or file_exclude_filter or file_install_tag:
             self.log.info(f'Remaining files after filtering: {len(mc.added) + len(mc.changed)}')
             # correct install size after filtering
             analysis_res.install_size = sum(fm.file_size for fm in manifest.file_manifest_list.elements

legendary/lfs/lgndry.py

@@ -46,6 +46,8 @@ class LGDLFS:
         # EOS Overlay install/update check info
         self._overlay_update_info = None
         self._overlay_install_info = None
+        # EGL content update check info
+        self._egl_content_update_info = None
         # Config with game specific settings (e.g. start parameters, env variables)
         self.config = LGDConf(comment_prefixes='/', allow_no_value=True)
 
@@ -401,19 +403,31 @@ class LGDLFS:
         json.dump(self._update_info, open(os.path.join(self.path, 'version.json'), 'w'),
                   indent=2, sort_keys=True)
 
-    def get_cached_sdl_data(self, app_name):
-        try:
-            return json.load(open(os.path.join(self.path, 'tmp', f'{app_name}.json')))
-        except Exception as e:
-            self.log.debug(f'Failed to load cached SDL data: {e!r}')
-            return None
+    @property
+    def egl_content_path(self) -> Path:
+        return Path(self.path) / 'egl_content'
 
-    def set_cached_sdl_data(self, app_name, sdl_version, sdl_data):
-        if not app_name or not sdl_data:
-            return
-        json.dump(dict(version=sdl_version, data=sdl_data),
-                  open(os.path.join(self.path, 'tmp', f'{app_name}.json'), 'w'),
-                  indent=2, sort_keys=True)
+    @property
+    def _egl_content_version_path(self) -> Path:
+        return Path(self.path) / 'egl_content_version.json'
+
+    def get_cached_egl_content_version(self):
+        if self._egl_content_update_info:
+            return self._egl_content_update_info
+
+        try:
+            self._egl_content_update_info = json.loads(self._egl_content_version_path.read_text())
+        except Exception as e:
+            self.log.debug(f'Failed to load EGL content version: {e!r}')
+            self._egl_content_update_info = dict(last_update=0, version=None)
+
+        return self._egl_content_update_info
+
+    def set_cached_egl_content_version(self, new_version: str):
+        self._egl_content_update_info = dict(version=new_version, last_update=time())
+        self._egl_content_version_path.write_text(
+            json.dumps(self._egl_content_update_info, indent=2, sort_keys=True)
+        )
 
     def get_cached_overlay_version(self):
         if self._overlay_update_info:

legendary/models/egl.py

@@ -59,6 +59,7 @@ class EGLManifest:
         self.install_location = None
         self.install_size = None
         self.install_tags = None
+        self.install_components = None
         self.installation_guid = None
         self.launch_command = None
         self.executable = None
@@ -87,6 +88,7 @@ class EGLManifest:
         tmp.install_location = json.pop('InstallLocation', '')
         tmp.install_size = json.pop('InstallSize', 0)
         tmp.install_tags = json.pop('InstallTags', [])
+        tmp.install_components = json.pop('InstallComponents', [])
         tmp.installation_guid = json.pop('InstallationGuid', '')
         tmp.launch_command = json.pop('LaunchCommand', '')
         tmp.executable = json.pop('LaunchExecutable', '')
@@ -114,6 +116,7 @@ class EGLManifest:
         out['InstallLocation'] = self.install_location
         out['InstallSize'] = self.install_size
         out['InstallTags'] = self.install_tags
+        out['InstallComponents'] = self.install_components
         out['InstallationGuid'] = self.installation_guid
         out['LaunchCommand'] = self.launch_command
         out['LaunchExecutable'] = self.executable
@@ -140,6 +143,7 @@ class EGLManifest:
         tmp.install_location = igame.install_path
         tmp.install_size = igame.install_size
         tmp.install_tags = igame.install_tags
+        tmp.install_components = igame.install_components
         tmp.installation_guid = igame.egl_guid
         tmp.launch_command = igame.launch_parameters
         tmp.executable = igame.executable
@@ -159,4 +163,4 @@ class EGLManifest:
                              launch_parameters=self.launch_command, can_run_offline=self.can_run_offline,
                              requires_ot=self.ownership_token, is_dlc=False,
                              needs_verification=self.needs_validation, install_size=self.install_size,
-                             egl_guid=self.installation_guid, install_tags=self.install_tags)
+                             egl_guid=self.installation_guid, install_tags=self.install_tags, install_components=self.install_components)

legendary/models/game.py

@@ -183,6 +183,7 @@ class InstalledGame:
     executable: str = ''
     install_size: int = 0
     install_tags: List[str] = field(default_factory=list)
+    install_components: List[str] = field(default_factory=list)
     is_dlc: bool = False
     launch_parameters: str = ''
     manifest_path: str = ''
@@ -218,6 +219,7 @@ class InstalledGame:
         tmp.install_size = json.get('install_size', 0)
         tmp.egl_guid = json.get('egl_guid', '')
         tmp.install_tags = json.get('install_tags', [])
+        tmp.install_components = json.get('install_components', [])
         return tmp
 
 

legendary/utils/cli.py

@@ -1,3 +1,10 @@
+from InquirerPy import inquirer
+from InquirerPy.base.control import Choice
+from InquirerPy.separator import Separator
+
+from epic_expreval import Tokenizer
+from legendary.utils.selective_dl import LGDEvaluationContext, EXTRA_FUNCTIONS
+
 def get_boolean_choice(prompt, default=True):
     yn = 'Y/n' if default else 'y/N'
 
@@ -43,33 +50,44 @@ def get_int_choice(prompt, default=None, min_choice=None, max_choice=None, retur
             return choice
 
 
-def sdl_prompt(sdl_data, title):
-    tags = ['']
-    if '__required' in sdl_data:
-        tags.extend(sdl_data['__required']['tags'])
-
+def sdl_prompt(sdl_data, title, context):
     print(f'You are about to install {title}, this application supports selective downloads.')
-    print('The following optional packs are available (tag - name):')
-    for tag, info in sdl_data.items():
-        if tag == '__required':
+    choices = []
+    required_categories = {}
+    for element in sdl_data['Data']:
+        is_required = element.get('IsRequired', 'false').lower() == 'true'
+        has_children = 'Children' in element
+        is_invisible = element.get('Invisible', 'false').lower() == 'true'
+        if (is_required and not has_children) or is_invisible:
             continue
-        print(' *', tag, '-', info['name'])
-
-    examples = ', '.join([g for g in sdl_data.keys() if g != '__required'][:2])
-    print(f'Please enter tags of pack(s) to install (space/comma-separated, e.g. "{examples}")')
-    print('Leave blank to use defaults (only required data will be downloaded).')
-    choices = input('Additional packs [Enter to confirm]: ')
-    if not choices:
-        return tags
-
-    for c in choices.strip('"').replace(',', ' ').split():
-        c = c.strip()
-        if c in sdl_data:
-            tags.extend(sdl_data[c]['tags'])
+        
+        if element.get('ConfigHandler'):
+            choices.append(Separator(4 * '-' + ' ' + element['Title'] + ' ' + 4 * '-'))
+            is_required = element.get('IsRequired', 'false').lower() == 'true'
+            if is_required: required_categories[element['UniqueId']] = []
+            for child in element.get('Children', []):
+                enabled = element.get('IsDefaultSelected', 'false').lower() == 'true'
+                choices.append(Choice(child['UniqueId'], name=child['Title'], enabled=enabled))
+                if is_required: required_categories[element['UniqueId']].append(child['UniqueId'])
         else:
-            print('Invalid tag:', c)
+            enabled = False
+            if element.get('IsDefaultSelected', 'false').lower() == 'true':
+                expression = element.get('DefaultSelectedExpression')
+                if expression:
+                    tk = Tokenizer(expression, context)
+                    tk.extend_functions(EXTRA_FUNCTIONS)
+                    tk.compile()
+                    if tk.execute(''):
+                        enabled = True
+                else:
+                    enabled = True
+            choices.append(Choice(element['UniqueId'], name=element['Title'], enabled=enabled))
 
-    return tags
+    selected_packs = inquirer.checkbox(message='Select optional packs to install',
+                                       choices=choices,
+                                       cycle=True,
+                                       validate=lambda selected: not required_categories or all(any(item in selected for item in category) for category in required_categories.values())).execute()
+    return selected_packs
 
 
 def strtobool(val):

legendary/utils/selective_dl.py

@@ -1,41 +1,93 @@
-# This file contains definitions for selective downloading for supported games
+# This file contains utilities for selective downloads in regards to parsing and evaluating sdlmeta
 # coding: utf-8
 
-_cyberpunk_sdl = {
-    'de': {'tags': ['voice_de_de'], 'name': 'Deutsch'},
-    'es': {'tags': ['voice_es_es'], 'name': 'español (España)'},
-    'fr': {'tags': ['voice_fr_fr'], 'name': 'français'},
-    'it': {'tags': ['voice_it_it'], 'name': 'italiano'},
-    'ja': {'tags': ['voice_ja_jp'], 'name': '日本語'},
-    'ko': {'tags': ['voice_ko_kr'], 'name': '한국어'},
-    'pl': {'tags': ['voice_pl_pl'], 'name': 'polski'},
-    'pt': {'tags': ['voice_pt_br'], 'name': 'português brasileiro'},
-    'ru': {'tags': ['voice_ru_ru'], 'name': 'русский'},
-    'cn': {'tags': ['voice_zh_cn'], 'name': '中文（中国）'}
+import os
+import json
+from epic_expreval import Tokenizer, EvaluationContext
+
+def has_access(context, app):
+    return bool(context.core.get_game(app))
+
+def is_selected(context, input):
+    return input in context.selection
+
+false_lambda = lambda c,i: False
+
+EXTRA_FUNCTIONS = {
+    'HasAccess': has_access,
+    "IsComponentSelected": is_selected,
+    "D3D12FeatureDataOptions1Check": false_lambda,
+    "D3D12FeatureDataOptions2Check": false_lambda,
+    "D3D12FeatureDataOptions3Check": false_lambda,
+    "D3D12FeatureDataOptions4Check": false_lambda,
+    "D3D12FeatureDataOptions5Check": false_lambda,
+    "D3D12FeatureDataOptions6Check": false_lambda,
+    "D3D12FeatureDataOptions7Check": false_lambda,
+    "D3D12FeatureDataOptions9Check": false_lambda,
+    "D3D12FeatureDataOptions9Check": false_lambda,
+    "IsIntelAtomic64EmulationSupported": false_lambda
 }
 
-_fortnite_sdl = {
-    '__required': {'tags': ['chunk0', 'chunk10'], 'name': 'Fortnite Core'},
-    'stw': {'tags': ['chunk11', 'chunk11optional'], 'name': 'Fortnite Save the World'},
-    'hd_textures': {'tags': ['chunk10optional'], 'name': 'High Resolution Textures'},
-    'lang_de': {'tags': ['chunk2'], 'name': '(Language Pack) Deutsch'},
-    'lang_fr': {'tags': ['chunk5'], 'name': '(Language Pack) français'},
-    'lang_pl': {'tags': ['chunk7'], 'name': '(Language Pack) polski'},
-    'lang_ru': {'tags': ['chunk8'], 'name': '(Language Pack) русский'},
-    'lang_cn': {'tags': ['chunk9'], 'name': '(Language Pack) 中文（中国）'}
-}
+class LGDEvaluationContext(EvaluationContext):
+    def __init__(self, core):
+        super().__init__()
+        self.core = core
+        self.selection = set()
 
-games = {
-    'Fortnite': _fortnite_sdl,
-    'Ginger': _cyberpunk_sdl
-}
+    def reset(self):
+        super().reset()
+        self.selection = set()
 
+def run_expression(expression, input):
+    """Runs expression with default EvauluationContext"""
+    tk = Tokenizer(expression, EvaluationContext())
+    tk.compile()
+    return tk.execute(input)
 
-def get_sdl_appname(app_name):
-    for k in games.keys():
-        if k.endswith('_Mac'):
-            continue
-        
-        if app_name.startswith(k):
-            return k
+def get_sdl_data(location, app_name, app_version):
+    applying_meta = []
+    for sdmeta_file in location.glob('*sdmeta'):
+        sdmeta = json.loads(sdmeta_file.read_text('utf-8-sig'))
+        is_applying_build = any(
+            build
+            for build in sdmeta.get('Builds')
+            if build.get('Asset') == app_name
+            and run_expression(build['Version'], app_version)
+        )
+        if is_applying_build:
+            applying_meta.append(sdmeta)
+            
+    if applying_meta:
+        return applying_meta[-1]
     return None
+
+def parse_components_selection(sdl_data, eval_context, install_components, install_tags):
+    for element in sdl_data['Data']:
+        if element.get('IsRequired', 'false').lower() == 'true':
+            install_tags.update(element.get('Tags', []))
+            if element['UniqueId'] not in install_components:
+                install_components.append(element['UniqueId'])
+            continue
+        if element.get('Invisible', 'false').lower() == 'true':
+            tk = Tokenizer(element['InvisibleSelectedExpression'], eval_context)
+            tk.extend_functions(EXTRA_FUNCTIONS)
+            tk.compile()
+            if tk.execute(''):
+                install_tags.update(element.get('Tags', []))
+                if element['UniqueId'] not in install_components:
+                    install_components.append(element['UniqueId'])
+
+        # The ids may change from revision to revision, this property lets us match options against older options
+        upgrade_id = element.get('UpgradePathLogic')
+        if upgrade_id and upgrade_id in install_components:
+            install_tags.update(element.get('Tags', []))
+            # Replace component id with upgraded one
+            install_components = [element['UniqueId'] if el == upgrade_id else el for el in install_components]
+
+        if element['UniqueId'] in install_components:
+            install_tags.update(element.get('Tags', []))
+
+        if element.get('ConfigHandler'):
+            for child in element.get('Children', []):
+                if child['UniqueId'] in install_components:
+                    install_tags.update(child.get('Tags', []))

requirements.txt

@@ -1,2 +1,4 @@
 requests<3.0
 filelock
+epic-expreval=0.2
+InquirerPy

setup.py

@@ -36,6 +36,8 @@ setup(
     ),
     install_requires=[
         'requests<3.0',
+        'epic-expreval==0.2',
+        'InquirerPy',
         'setuptools',
         'wheel',
         'filelock'