From 4c8a6e84e7301ddcf5e78f25aeb7d0abd1cad9a7 Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 16 Jul 2025 18:28:23 +0200 Subject: [PATCH 01/33] WIP test click CI wrapper --- easybuild/cli/__init__.py | 37 ++++++++ easybuild/cli/options/__init__.py | 150 ++++++++++++++++++++++++++++++ eb2 | 8 ++ setup.py | 1 + 4 files changed, 196 insertions(+) create mode 100644 easybuild/cli/__init__.py create mode 100644 easybuild/cli/options/__init__.py create mode 100644 eb2 diff --git a/easybuild/cli/__init__.py b/easybuild/cli/__init__.py new file mode 100644 index 0000000000..f759603645 --- /dev/null +++ b/easybuild/cli/__init__.py @@ -0,0 +1,37 @@ +try: + import rich_click as click +except ImportError: + import click + +try: + from rich.traceback import install +except ImportError: + pass +else: + install(suppress=[click]) + +from .options import EasyBuildCliOption + +from easybuild.main import main_with_hooks + +@click.command() +@EasyBuildCliOption.apply_options +@click.pass_context +@click.argument('other_args', nargs=-1, type=click.UNPROCESSED, required=False) +def eb(ctx, other_args): + """EasyBuild command line interface.""" + args = [] + for key, value in ctx.hidden_params.items(): + key = key.replace('_', '-') + if isinstance(value, bool): + if value: + args.append(f"--{key}") + else: + if value and value != EasyBuildCliOption.OPTIONS_MAP[key].default: + if isinstance(value, (list, tuple)): + value = ','.join(value) + args.append(f"--{key}={value}") + + args.extend(other_args) + + main_with_hooks(args=args) diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py new file mode 100644 index 0000000000..89577a89bb --- /dev/null +++ b/easybuild/cli/options/__init__.py @@ -0,0 +1,150 @@ +import os + +from typing import Callable, Any +from dataclasses import dataclass + +opt_group = {} +try: + import rich_click as click +except ImportError: + import click +else: + opt_group = click.rich_click.OPTION_GROUPS + +from easybuild.tools.options import EasyBuildOptions + +DEBUG_EASYBUILD_OPTIONS = os.environ.get('DEBUG_EASYBUILD_OPTIONS', '').lower() in ('1', 'true', 'yes', 'y') + +class OptionExtracter(EasyBuildOptions): + def __init__(self, *args, **kwargs): + self._option_dicts = {} + super().__init__(*args, **kwargs) + + def add_group_parser(self, opt_dict, descr, *args, prefix='', **kwargs): + super().add_group_parser(opt_dict, descr, *args, prefix=prefix, **kwargs) + self._option_dicts[descr[0]] = (prefix, opt_dict) + +extracter = OptionExtracter(go_args=[]) + +def register_hidden_param(ctx, param, value): + """Register a hidden parameter in the context.""" + if not hasattr(ctx, 'hidden_params'): + ctx.hidden_params = {} + ctx.hidden_params[param.name] = value + +@dataclass +class OptionData: + name: str + description: str + type: str + action: str + default: Any + group: str = None + short: str = None + meta: dict = None + lst: list = None + + def __post_init__(self): + if self.short is not None and not isinstance(self.short, str): + raise TypeError(f"Short option must be a string, got {type(self.short)}") + if self.meta is not None and not isinstance(self.meta, dict): + raise TypeError(f"Meta must be a dictionary, got {type(self.meta)}") + if self.lst is not None and not isinstance(self.lst, (list, tuple)): + raise TypeError(f"List must be a list or tuple, got {type(self.lst)}") + + def to_click_option_dec(self): + """Convert OptionData to a click.Option.""" + decls = [f"--{self.name}"] + if self.short: + decls.insert(0, f"-{self.short}") + + kwargs = { + 'help': self.description, + # 'help': '123', + 'default': self.default, + 'show_default': True, + } + + if self.default is False or self.default is True: + kwargs['is_flag'] = True + + if isinstance(self.default, (list, tuple)): + kwargs['multiple'] = True + kwargs['type'] = click.STRING + + return click.option( + *decls, + expose_value=False, + callback=register_hidden_param, + **kwargs + ) + +class EasyBuildCliOption(): + OPTIONS: list[OptionData] = [] + OPTIONS_MAP: dict[str, OptionData] = {} + + @classmethod + def apply_options(cls, function: Callable) -> Callable: + """Decorator to apply EasyBuild options to a function.""" + group_data = {} + for opt_obj in cls.OPTIONS: + group_data.setdefault(opt_obj.group, []).append(f'--{opt_obj.name}') + function = opt_obj.to_click_option_dec()(function) + lst = [] + for key, value in group_data.items(): + lst.append({ + 'name': key, + # 'description': f'Options for {key}', + 'options': value + }) + opt_group[function.__name__] = lst + return function + + @classmethod + def register_option(cls, group: str, name: str, data: tuple, prefix: str = '') -> None: + """Register an EasyBuild option.""" + if prefix: + name = f"{prefix}-{name}" + if name == 'help': + return + short = None + meta = None + lst = None + descr, typ, action, default, *others = data + while others: + opt = others.pop(0) + if isinstance(opt, str): + if short is not None: + raise ValueError(f"Short option already set: {short} for {name}") + short = opt + elif isinstance(opt, dict): + if meta is not None: + raise ValueError(f"Meta already set: {meta} for {name}") + meta = opt + elif isinstance(opt, (list, tuple)): + if lst is not None: + raise ValueError(f"List already set: {lst} for {name}") + lst = opt + else: + raise ValueError(f"Unexpected type for others: {type(others[0])} in {others}") + + opt = OptionData( + group=group, + name=name, + description=descr, + type=typ, + action=action, + default=default, + short=short, + meta=meta, + lst=lst + ) + cls.OPTIONS_MAP[name] = opt + cls.OPTIONS.append(opt) + +for grp, dct in extracter._option_dicts.items(): + prefix, dct = dct + if dct is None: + continue + for key, value in dct.items(): + EasyBuildCliOption.register_option(grp, key, value, prefix=prefix) diff --git a/eb2 b/eb2 new file mode 100644 index 0000000000..94cb0927bd --- /dev/null +++ b/eb2 @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import re +import sys +from easybuild.cli import eb +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(eb()) diff --git a/setup.py b/setup.py index 6cdff71f56..5c0ab8ca2f 100644 --- a/setup.py +++ b/setup.py @@ -111,6 +111,7 @@ def find_rel_test(): package_data={'test.framework': find_rel_test()}, scripts=[ 'eb', + 'eb2', # bash completion 'optcomplete.bash', 'minimal_bash_completion.bash', From 0ff09ebe4fdbb2af2e2815e44c205b494012fa84 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 17 Jul 2025 10:45:11 +0200 Subject: [PATCH 02/33] Install as a console script --- eb2 | 8 -------- setup.py | 6 +++++- 2 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 eb2 diff --git a/eb2 b/eb2 deleted file mode 100644 index 94cb0927bd..0000000000 --- a/eb2 +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import re -import sys -from easybuild.cli import eb -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(eb()) diff --git a/setup.py b/setup.py index 5c0ab8ca2f..0c29403c69 100644 --- a/setup.py +++ b/setup.py @@ -111,7 +111,6 @@ def find_rel_test(): package_data={'test.framework': find_rel_test()}, scripts=[ 'eb', - 'eb2', # bash completion 'optcomplete.bash', 'minimal_bash_completion.bash', @@ -120,6 +119,11 @@ def find_rel_test(): # utility scripts 'easybuild/scripts/install_eb_dep.sh', ], + entry_points={ + 'console_scripts': [ + 'eb2 = easybuild.cli:eb', + ] + }, data_files=[ ('easybuild/scripts', glob.glob('easybuild/scripts/*')), ('etc', glob.glob('etc/*')), From 60229a0db4e4c17d35ebfbf8919827fa596259ce Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 17 Jul 2025 10:45:39 +0200 Subject: [PATCH 03/33] Avoid warning of double arg initialization if `prepare_main` did not error --- easybuild/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/main.py b/easybuild/main.py index 034a9e9dd2..00ea3a19a2 100755 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -837,6 +837,8 @@ def main_with_hooks(args=None): init_session_state, eb_go, cfg_settings = prepare_main(args=args) except EasyBuildError as err: print_error(err.msg, exit_code=err.exit_code) + else: + args = None hooks = load_hooks(eb_go.options.hooks) From b6990fc92dcf0d95ff3f61b0f45d41911a2bd00f Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 17 Jul 2025 10:45:50 +0200 Subject: [PATCH 04/33] Move to static method --- easybuild/cli/options/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py index 89577a89bb..2c70c08b4b 100644 --- a/easybuild/cli/options/__init__.py +++ b/easybuild/cli/options/__init__.py @@ -26,11 +26,6 @@ def add_group_parser(self, opt_dict, descr, *args, prefix='', **kwargs): extracter = OptionExtracter(go_args=[]) -def register_hidden_param(ctx, param, value): - """Register a hidden parameter in the context.""" - if not hasattr(ctx, 'hidden_params'): - ctx.hidden_params = {} - ctx.hidden_params[param.name] = value @dataclass class OptionData: @@ -75,10 +70,17 @@ def to_click_option_dec(self): return click.option( *decls, expose_value=False, - callback=register_hidden_param, + callback=self.register_hidden_param, **kwargs ) + @staticmethod + def register_hidden_param(ctx, param, value): + """Register a hidden parameter in the context.""" + if not hasattr(ctx, 'hidden_params'): + ctx.hidden_params = {} + ctx.hidden_params[param.name] = value + class EasyBuildCliOption(): OPTIONS: list[OptionData] = [] OPTIONS_MAP: dict[str, OptionData] = {} From 08b0365c13c211fb4a36a2b01cfe956a6cdf196e Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 18 Jul 2025 13:37:33 +0200 Subject: [PATCH 05/33] Added list paramter conversion and better autocomplete --- easybuild/cli/__init__.py | 27 ++++- easybuild/cli/options/__init__.py | 175 ++++++++++++++++++++++++++++-- 2 files changed, 190 insertions(+), 12 deletions(-) diff --git a/easybuild/cli/__init__.py b/easybuild/cli/__init__.py index f759603645..d05e5f2a95 100644 --- a/easybuild/cli/__init__.py +++ b/easybuild/cli/__init__.py @@ -1,19 +1,26 @@ +import os + try: import rich_click as click + import click as original_click except ImportError: import click + import click as original_click try: from rich.traceback import install except ImportError: pass else: - install(suppress=[click]) + install(suppress=[ + click, original_click + ]) from .options import EasyBuildCliOption from easybuild.main import main_with_hooks + @click.command() @EasyBuildCliOption.apply_options @click.pass_context @@ -27,9 +34,21 @@ def eb(ctx, other_args): if value: args.append(f"--{key}") else: - if value and value != EasyBuildCliOption.OPTIONS_MAP[key].default: - if isinstance(value, (list, tuple)): - value = ','.join(value) + opt = EasyBuildCliOption.OPTIONS_MAP[key] + if value and value != opt.default: + if isinstance(value, (list, tuple)) and value: + if isinstance(value[0], list): + value = sum(value, []) + if 'path' in opt.type: + delim = os.pathsep + elif 'str' in opt.type: + delim = ',' + elif 'url' in opt.type: + delim = '|' + else: + raise ValueError(f"Unsupported type for {key}: {opt.type}") + value = delim.join(value) + print(f"--Adding {key}={value} to args") args.append(f"--{key}={value}") args.extend(other_args) diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py index 2c70c08b4b..d9402403ee 100644 --- a/easybuild/cli/options/__init__.py +++ b/easybuild/cli/options/__init__.py @@ -1,8 +1,12 @@ +# import logging import os from typing import Callable, Any from dataclasses import dataclass +from click.shell_completion import CompletionItem +from easybuild.tools.options import EasyBuildOptions + opt_group = {} try: import rich_click as click @@ -10,10 +14,8 @@ import click else: opt_group = click.rich_click.OPTION_GROUPS + opt_group.clear() # Clear existing groups to avoid conflicts -from easybuild.tools.options import EasyBuildOptions - -DEBUG_EASYBUILD_OPTIONS = os.environ.get('DEBUG_EASYBUILD_OPTIONS', '').lower() in ('1', 'true', 'yes', 'y') class OptionExtracter(EasyBuildOptions): def __init__(self, *args, **kwargs): @@ -24,9 +26,69 @@ def add_group_parser(self, opt_dict, descr, *args, prefix='', **kwargs): super().add_group_parser(opt_dict, descr, *args, prefix=prefix, **kwargs) self._option_dicts[descr[0]] = (prefix, opt_dict) + extracter = OptionExtracter(go_args=[]) +class DelimitedPathList(click.Path): + """Custom Click parameter type for delimited lists.""" + name = 'pathlist' + + def __init__(self, *args, delimiter=',', resolve_full: bool = True, **kwargs): + super().__init__(*args, **kwargs) + self.delimiter = delimiter + self.resolve_full = resolve_full + + def convert(self, value, param, ctx): + if not isinstance(value, str): + raise click.BadParameter(f"Expected a comma-separated string, got {value}") + res = value.split(self.delimiter) + if self.resolve_full: + res = [os.path.abspath(v) for v in res] + return res + + def shell_complete(self, ctx, param, incomplete): + others, last = ([''] + incomplete.rsplit(self.delimiter, 1))[-2:] + # logging.warning(f"Shell completion for delimited path list: others={others}, last={last}") + dir_path, prefix = os.path.split(last) + dir_path = dir_path or '.' + # logging.warning(f"Shell completion for delimited path list: dir_path={dir_path}, prefix={prefix}") + possibles = [] + for path in os.listdir(dir_path): + if not path.startswith(prefix): + continue + full_path = os.path.join(dir_path, path) + if os.path.isdir(full_path): + if self.dir_okay: + possibles.append(full_path) + possibles.append(full_path + os.sep) + elif os.path.isfile(full_path): + if self.file_okay: + possibles.append(full_path) + start = f'{others}{self.delimiter}' if others else '' + res = [CompletionItem(f"{start}{path}") for path in possibles] + # logging.warning(f"Shell completion for delimited path list: res={possibles}") + return res + + +class DelimitedString(click.ParamType): + """Custom Click parameter type for delimited strings.""" + name = 'strlist' + + def __init__(self, *args, delimiter=',', **kwargs): + super().__init__(*args, **kwargs) + self.delimiter = delimiter + + def convert(self, value, param, ctx): + if isinstance(value, str): + return value.split(self.delimiter) + raise click.BadParameter(f"Expected a string or a comma-separated string, got {value}") + + def shell_complete(self, ctx, param, incomplete): + last = incomplete.rsplit(self.delimiter, 1)[-1] + return super().shell_complete(ctx, param, last) + + @dataclass class OptionData: name: str @@ -55,17 +117,112 @@ def to_click_option_dec(self): kwargs = { 'help': self.description, - # 'help': '123', 'default': self.default, 'show_default': True, + 'type': None } - if self.default is False or self.default is True: - kwargs['is_flag'] = True - - if isinstance(self.default, (list, tuple)): + if self.type in ['strlist', 'strtuple']: + kwargs['type'] = DelimitedString(delimiter=',') kwargs['multiple'] = True + elif self.type in ['pathlist', 'pathtuple']: + # kwargs['type'] = DelimitedPathList(delimiter=os.pathsep) + kwargs['type'] = DelimitedPathList(delimiter=',') + kwargs['multiple'] = True + elif self.type in ['urllist', 'urltuple']: + kwargs['type'] = DelimitedString(delimiter='|') + kwargs['multiple'] = True + elif self.type == 'choice': + if self.lst is None: + raise ValueError(f"Choice type requires a list of choices for option {self.name}") + kwargs['type'] = click.Choice(self.lst, case_sensitive=False) + elif self.type in ['int', int]: + kwargs['type'] = click.INT + elif self.type in ['float', float]: + kwargs['type'] = click.FLOAT + elif self.type in ['str', str]: kwargs['type'] = click.STRING + elif self.type is None: + if self.default is False or self.default is True: + kwargs['is_flag'] = True + kwargs['type'] = click.BOOL + elif isinstance(self.default, (list, tuple)): + kwargs['multiple'] = True + kwargs['type'] = click.STRING + + # if kwargs['type'] is None: + # print(f"Warning: No type specified for option {self.name}, defaulting to STRING") + + # actions = set() + # for opt in EasyBuildCliOption.OPTIONS: + # actions.add(opt.action) + # print(f"Registered {len(EasyBuildCliOption.OPTIONS)} options with actions: {actions}") + # # Registered 296 options with actions: { + # # 'store_infolog', 'add_flex', 'append', 'add', 'store_true', 'store_debuglog', 'store_or_None', + # # 'store_warninglog', 'store', 'extend', 'regex' + # # } + + # Actions: + # - shorthelp : hook for shortend help messages + # - confighelp : hook for configfile-style help messages + # - store_debuglog : turns on fancylogger debugloglevel + # - also: 'store_infolog', 'store_warninglog' + # - add : add value to default (result is default + value) + # - add_first : add default to value (result is value + default) + # - extend : alias for add with strlist type + # - type must support + (__add__) and one of negate (__neg__) or slicing (__getslice__) + # - add_flex : similar to add / add_first, but replaces the first "empty" element with the default + # - the empty element is dependent of the type + # - for {str,path}{list,tuple} this is the empty string + # - types must support the index method to determine the location of the "empty" element + # - the replacement uses + + # - e.g. a strlist type with value "0,,1"` and default [3,4] and action add_flex will + # use the empty string '' as "empty" element, and will result in [0,3,4,1] (not [0,[3,4],1]) + # (but also a strlist with value "" and default [3,4] will result in [3,4]; + # so you can't set an empty list with add_flex) + # - date : convert into datetime.date + # - datetime : convert into datetime.datetime + # - regex: compile str in regexp + # - store_or_None + # - set default to None if no option passed, + # - set to default if option without value passed, + # - set to value if option with value passed + + # Types: + # - strlist, strtuple : convert comma-separated string in a list resp. tuple of strings + # - pathlist, pathtuple : using os.pathsep, convert pathsep-separated string in a list resp. tuple of strings + # - the path separator is OS-dependent + # - urllist, urltuple: convert string seperated by '|' to a list resp. tuple of strings + + # def take_action(self, action, dest, opt, value, values, parser): + # if action == "store": + # setattr(values, dest, value) + # elif action == "store_const": + # setattr(values, dest, self.const) + # elif action == "store_true": + # setattr(values, dest, True) + # elif action == "store_false": + # setattr(values, dest, False) + # elif action == "append": + # values.ensure_value(dest, []).append(value) + # elif action == "append_const": + # values.ensure_value(dest, []).append(self.const) + # elif action == "count": + # setattr(values, dest, values.ensure_value(dest, 0) + 1) + # elif action == "callback": + # args = self.callback_args or () + # kwargs = self.callback_kwargs or {} + # self.callback(self, opt, value, parser, *args, **kwargs) + # elif action == "help": + # parser.print_help() + # parser.exit() + # elif action == "version": + # parser.print_version() + # parser.exit() + # else: + # raise ValueError("unknown action %r" % self.action) + + # return 1 return click.option( *decls, @@ -81,6 +238,7 @@ def register_hidden_param(ctx, param, value): ctx.hidden_params = {} ctx.hidden_params[param.name] = value + class EasyBuildCliOption(): OPTIONS: list[OptionData] = [] OPTIONS_MAP: dict[str, OptionData] = {} @@ -144,6 +302,7 @@ def register_option(cls, group: str, name: str, data: tuple, prefix: str = '') - cls.OPTIONS_MAP[name] = opt cls.OPTIONS.append(opt) + for grp, dct in extracter._option_dicts.items(): prefix, dct = dct if dct is None: From eb7af7210fa1adc81897d16a8d66fce6ac8730c4 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 18 Jul 2025 13:38:33 +0200 Subject: [PATCH 06/33] Removed comments --- easybuild/cli/options/__init__.py | 74 ------------------------------- 1 file changed, 74 deletions(-) diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py index d9402403ee..935adf2d41 100644 --- a/easybuild/cli/options/__init__.py +++ b/easybuild/cli/options/__init__.py @@ -150,80 +150,6 @@ def to_click_option_dec(self): kwargs['multiple'] = True kwargs['type'] = click.STRING - # if kwargs['type'] is None: - # print(f"Warning: No type specified for option {self.name}, defaulting to STRING") - - # actions = set() - # for opt in EasyBuildCliOption.OPTIONS: - # actions.add(opt.action) - # print(f"Registered {len(EasyBuildCliOption.OPTIONS)} options with actions: {actions}") - # # Registered 296 options with actions: { - # # 'store_infolog', 'add_flex', 'append', 'add', 'store_true', 'store_debuglog', 'store_or_None', - # # 'store_warninglog', 'store', 'extend', 'regex' - # # } - - # Actions: - # - shorthelp : hook for shortend help messages - # - confighelp : hook for configfile-style help messages - # - store_debuglog : turns on fancylogger debugloglevel - # - also: 'store_infolog', 'store_warninglog' - # - add : add value to default (result is default + value) - # - add_first : add default to value (result is value + default) - # - extend : alias for add with strlist type - # - type must support + (__add__) and one of negate (__neg__) or slicing (__getslice__) - # - add_flex : similar to add / add_first, but replaces the first "empty" element with the default - # - the empty element is dependent of the type - # - for {str,path}{list,tuple} this is the empty string - # - types must support the index method to determine the location of the "empty" element - # - the replacement uses + - # - e.g. a strlist type with value "0,,1"` and default [3,4] and action add_flex will - # use the empty string '' as "empty" element, and will result in [0,3,4,1] (not [0,[3,4],1]) - # (but also a strlist with value "" and default [3,4] will result in [3,4]; - # so you can't set an empty list with add_flex) - # - date : convert into datetime.date - # - datetime : convert into datetime.datetime - # - regex: compile str in regexp - # - store_or_None - # - set default to None if no option passed, - # - set to default if option without value passed, - # - set to value if option with value passed - - # Types: - # - strlist, strtuple : convert comma-separated string in a list resp. tuple of strings - # - pathlist, pathtuple : using os.pathsep, convert pathsep-separated string in a list resp. tuple of strings - # - the path separator is OS-dependent - # - urllist, urltuple: convert string seperated by '|' to a list resp. tuple of strings - - # def take_action(self, action, dest, opt, value, values, parser): - # if action == "store": - # setattr(values, dest, value) - # elif action == "store_const": - # setattr(values, dest, self.const) - # elif action == "store_true": - # setattr(values, dest, True) - # elif action == "store_false": - # setattr(values, dest, False) - # elif action == "append": - # values.ensure_value(dest, []).append(value) - # elif action == "append_const": - # values.ensure_value(dest, []).append(self.const) - # elif action == "count": - # setattr(values, dest, values.ensure_value(dest, 0) + 1) - # elif action == "callback": - # args = self.callback_args or () - # kwargs = self.callback_kwargs or {} - # self.callback(self, opt, value, parser, *args, **kwargs) - # elif action == "help": - # parser.print_help() - # parser.exit() - # elif action == "version": - # parser.print_version() - # parser.exit() - # else: - # raise ValueError("unknown action %r" % self.action) - - # return 1 - return click.option( *decls, expose_value=False, From 12dfbf6d31ab848dc8bb9630a0223badad15af91 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 18 Jul 2025 13:39:39 +0200 Subject: [PATCH 07/33] Fix potential missing attribute --- easybuild/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/cli/__init__.py b/easybuild/cli/__init__.py index d05e5f2a95..c8c6c63854 100644 --- a/easybuild/cli/__init__.py +++ b/easybuild/cli/__init__.py @@ -28,7 +28,7 @@ def eb(ctx, other_args): """EasyBuild command line interface.""" args = [] - for key, value in ctx.hidden_params.items(): + for key, value in getattr(ctx, 'hidden_params', {}).items(): key = key.replace('_', '-') if isinstance(value, bool): if value: From 5109e31b19d776d312a3fef942228830721ace15 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 18 Jul 2025 14:30:46 +0200 Subject: [PATCH 08/33] Added autocompletion for EC files --- easybuild/cli/__init__.py | 8 +++++--- easybuild/cli/options/__init__.py | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/easybuild/cli/__init__.py b/easybuild/cli/__init__.py index c8c6c63854..2be53f66f4 100644 --- a/easybuild/cli/__init__.py +++ b/easybuild/cli/__init__.py @@ -16,7 +16,7 @@ click, original_click ]) -from .options import EasyBuildCliOption +from .options import EasyBuildCliOption, EasyconfigParam from easybuild.main import main_with_hooks @@ -24,17 +24,19 @@ @click.command() @EasyBuildCliOption.apply_options @click.pass_context -@click.argument('other_args', nargs=-1, type=click.UNPROCESSED, required=False) +@click.argument('other_args', nargs=-1, type=EasyconfigParam(), required=False) def eb(ctx, other_args): """EasyBuild command line interface.""" args = [] for key, value in getattr(ctx, 'hidden_params', {}).items(): key = key.replace('_', '-') + opt = EasyBuildCliOption.OPTIONS_MAP[key] + if value in ['False', 'True']: + value = value == 'True' if isinstance(value, bool): if value: args.append(f"--{key}") else: - opt = EasyBuildCliOption.OPTIONS_MAP[key] if value and value != opt.default: if isinstance(value, (list, tuple)) and value: if isinstance(value[0], list): diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py index 935adf2d41..8a0ea01edb 100644 --- a/easybuild/cli/options/__init__.py +++ b/easybuild/cli/options/__init__.py @@ -5,7 +5,9 @@ from dataclasses import dataclass from click.shell_completion import CompletionItem -from easybuild.tools.options import EasyBuildOptions +from easybuild.tools.options import EasyBuildOptions, set_up_configuration +from easybuild.tools.robot import search_easyconfigs + opt_group = {} try: @@ -89,6 +91,17 @@ def shell_complete(self, ctx, param, incomplete): return super().shell_complete(ctx, param, last) +class EasyconfigParam(click.ParamType): + """Custom Click parameter type for easyconfig parameters.""" + name = 'easyconfig' + + def shell_complete(self, ctx, param, incomplete): + if not incomplete: + return [] + set_up_configuration(args=["--ignore-index"], silent=True, reconfigure=True) + return [CompletionItem(ec) for ec in search_easyconfigs(fr'^{incomplete}.*\.eb$', filename_only=True)] + + @dataclass class OptionData: name: str From f3972f8107bcb0092067bfec89cd01873844935e Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 18 Jul 2025 14:44:09 +0200 Subject: [PATCH 09/33] Better checks for default values --- easybuild/cli/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/easybuild/cli/__init__.py b/easybuild/cli/__init__.py index 2be53f66f4..d3c9d63e81 100644 --- a/easybuild/cli/__init__.py +++ b/easybuild/cli/__init__.py @@ -37,10 +37,15 @@ def eb(ctx, other_args): if value: args.append(f"--{key}") else: + if isinstance(value, (list, tuple)) and value: + # Flatten nested lists if necessary + if isinstance(value[0], list): + value = sum(value, []) + # Match the type of the option with the default to see if we need to add it + if isinstance(value, list) and isinstance(opt.default, tuple): + value = tuple(value) if value and value != opt.default: - if isinstance(value, (list, tuple)) and value: - if isinstance(value[0], list): - value = sum(value, []) + if isinstance(value, (list, tuple)): if 'path' in opt.type: delim = os.pathsep elif 'str' in opt.type: @@ -50,7 +55,8 @@ def eb(ctx, other_args): else: raise ValueError(f"Unsupported type for {key}: {opt.type}") value = delim.join(value) - print(f"--Adding {key}={value} to args") + + # print(f"--Adding {key}={value} to args") args.append(f"--{key}={value}") args.extend(other_args) From 530db7331b4fc6e604ed719b4bc33a192b3aef0d Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 18 Jul 2025 16:34:59 +0200 Subject: [PATCH 10/33] - Fixed behavior of bool --X/--disable-X - Fixed beahvore of defaults with `store_or_None` - Fixed checking defaults vs list/tuple instead of flattened value - Added checks for list values in covert comming from the defaults --- easybuild/cli/__init__.py | 61 ++++++++++++++++++++++--------- easybuild/cli/options/__init__.py | 34 +++++++++++++---- 2 files changed, 70 insertions(+), 25 deletions(-) diff --git a/easybuild/cli/__init__.py b/easybuild/cli/__init__.py index d3c9d63e81..4dfdb74ec3 100644 --- a/easybuild/cli/__init__.py +++ b/easybuild/cli/__init__.py @@ -31,34 +31,61 @@ def eb(ctx, other_args): for key, value in getattr(ctx, 'hidden_params', {}).items(): key = key.replace('_', '-') opt = EasyBuildCliOption.OPTIONS_MAP[key] + # TEST_KEYS = [ + # 'robot', + # 'robot-paths', + # 'map-toolchains', + # 'search-paths', + # ] + # if key in TEST_KEYS: + # print(f'{key.upper()}: `{value=}` `{type(value)=}` `{opt.default=}` `{opt.action=}`') if value in ['False', 'True']: value = value == 'True' if isinstance(value, bool): - if value: - args.append(f"--{key}") + if value != opt.default: + if value: + args.append(f"--{key}") + else: + args.append(f"--disable-{key}") else: + if isinstance(value, (list, tuple)) and value: # Flatten nested lists if necessary if isinstance(value[0], list): value = sum(value, []) # Match the type of the option with the default to see if we need to add it - if isinstance(value, list) and isinstance(opt.default, tuple): + if value and isinstance(value, list) and isinstance(opt.default, tuple): value = tuple(value) - if value and value != opt.default: - if isinstance(value, (list, tuple)): - if 'path' in opt.type: - delim = os.pathsep - elif 'str' in opt.type: - delim = ',' - elif 'url' in opt.type: - delim = '|' - else: - raise ValueError(f"Unsupported type for {key}: {opt.type}") - value = delim.join(value) + if value and isinstance(value, tuple) and isinstance(opt.default, list): + value = list(value) + value_is_default = (value == opt.default) - # print(f"--Adding {key}={value} to args") - args.append(f"--{key}={value}") + value_flattened = value + if isinstance(value, (list, tuple)): + if 'path' in opt.type: + delim = os.pathsep + elif 'str' in opt.type: + delim = ',' + elif 'url' in opt.type: + delim = '|' + else: + raise ValueError(f"Unsupported type for {key}: {opt.type}") + value_flattened = delim.join(value) - args.extend(other_args) + if opt.action == 'store_or_None': + if value is None or value == (): + continue + if value_is_default: + args.append(f"--{key}") + else: + args.append(f"--{key}={value_flattened}") + elif value and not value_is_default: + if value: + args.append(f"--{key}={value_flattened}") + else: + args.append(f"--{key}") + # for arg in args: + # print(f"ARG: {arg}") + args.extend(other_args) main_with_hooks(args=args) diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py index 8a0ea01edb..8f974b5102 100644 --- a/easybuild/cli/options/__init__.py +++ b/easybuild/cli/options/__init__.py @@ -42,11 +42,16 @@ def __init__(self, *args, delimiter=',', resolve_full: bool = True, **kwargs): self.resolve_full = resolve_full def convert(self, value, param, ctx): - if not isinstance(value, str): + # logging.warning(f"{param=} convert called with `{value=}`, `{type(value)=}`") + if isinstance(value, str): + res = value.split(self.delimiter) + elif isinstance(value, (list, tuple)): + res = value + else: raise click.BadParameter(f"Expected a comma-separated string, got {value}") - res = value.split(self.delimiter) if self.resolve_full: res = [os.path.abspath(v) for v in res] + # logging.warning(f"{param=} convert returning `{res=}`") return res def shell_complete(self, ctx, param, incomplete): @@ -83,8 +88,12 @@ def __init__(self, *args, delimiter=',', **kwargs): def convert(self, value, param, ctx): if isinstance(value, str): - return value.split(self.delimiter) - raise click.BadParameter(f"Expected a string or a comma-separated string, got {value}") + res = value.split(self.delimiter) + elif isinstance(value, (list, tuple)): + res = value + else: + raise click.BadParameter(f"Expected a string or a comma-separated string, got {value}") + return res def shell_complete(self, ctx, param, incomplete): last = incomplete.rsplit(self.delimiter, 1)[-1] @@ -124,13 +133,15 @@ def __post_init__(self): def to_click_option_dec(self): """Convert OptionData to a click.Option.""" - decls = [f"--{self.name}"] + decl = f"--{self.name}" + other_decls = [] if self.short: - decls.insert(0, f"-{self.short}") + other_decls.insert(0, f"-{self.short}") kwargs = { 'help': self.description, 'default': self.default, + 'is_flag': False, 'show_default': True, 'type': None } @@ -139,7 +150,6 @@ def to_click_option_dec(self): kwargs['type'] = DelimitedString(delimiter=',') kwargs['multiple'] = True elif self.type in ['pathlist', 'pathtuple']: - # kwargs['type'] = DelimitedPathList(delimiter=os.pathsep) kwargs['type'] = DelimitedPathList(delimiter=',') kwargs['multiple'] = True elif self.type in ['urllist', 'urltuple']: @@ -148,7 +158,7 @@ def to_click_option_dec(self): elif self.type == 'choice': if self.lst is None: raise ValueError(f"Choice type requires a list of choices for option {self.name}") - kwargs['type'] = click.Choice(self.lst, case_sensitive=False) + kwargs['type'] = click.Choice(self.lst, case_sensitive=True) elif self.type in ['int', int]: kwargs['type'] = click.INT elif self.type in ['float', float]: @@ -159,10 +169,18 @@ def to_click_option_dec(self): if self.default is False or self.default is True: kwargs['is_flag'] = True kwargs['type'] = click.BOOL + if self.action in ['store_true', 'store_false']: + decl = f"--{self.name}/--disable-{self.name}" elif isinstance(self.default, (list, tuple)): kwargs['multiple'] = True kwargs['type'] = click.STRING + if self.action == 'store_or_None': + kwargs['default'] = None + kwargs['flag_value'] = self.default + + decls = other_decls + [decl] + return click.option( *decls, expose_value=False, From b42d59aec9d96b6d3489482d4b095c4950d65129 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 31 Jul 2025 13:16:24 +0200 Subject: [PATCH 11/33] Do not resolve full path to avoid `:` in `robot-paths` being improperly converted --- easybuild/cli/options/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py index 8f974b5102..a1b7c18041 100644 --- a/easybuild/cli/options/__init__.py +++ b/easybuild/cli/options/__init__.py @@ -36,7 +36,7 @@ class DelimitedPathList(click.Path): """Custom Click parameter type for delimited lists.""" name = 'pathlist' - def __init__(self, *args, delimiter=',', resolve_full: bool = True, **kwargs): + def __init__(self, *args, delimiter=',', resolve_full: bool = False, **kwargs): super().__init__(*args, **kwargs) self.delimiter = delimiter self.resolve_full = resolve_full From 8f6e8631b5da86965289d7d9e8439d97650ed591 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 31 Jul 2025 13:54:33 +0200 Subject: [PATCH 12/33] Ensure that if `click` is not present a nicer message is shown --- easybuild/cli/__init__.py | 154 +++++++++++++++++++------------------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/easybuild/cli/__init__.py b/easybuild/cli/__init__.py index 4dfdb74ec3..dd574adb1f 100644 --- a/easybuild/cli/__init__.py +++ b/easybuild/cli/__init__.py @@ -1,91 +1,91 @@ import os +import sys try: - import rich_click as click import click as original_click except ImportError: - import click - import click as original_click - -try: - from rich.traceback import install -except ImportError: - pass + def eb(): + """Placeholder function to inform the user that `click` is required.""" + print("Using `eb2` requires `click` to be installed. Either use `eb` or install `click` with `pip install click`.") + print("`eb2` also uses `rich` and `rich_click` as optional dependencies for enhanced CLI experience.") + print("Exiting...") + sys.exit(0) else: - install(suppress=[ - click, original_click - ]) + try: + import rich_click as click + except ImportError: + import click -from .options import EasyBuildCliOption, EasyconfigParam + try: + from rich.traceback import install + except ImportError: + pass + else: + install(suppress=[ + click, original_click + ]) -from easybuild.main import main_with_hooks + from .options import EasyBuildCliOption, EasyconfigParam + from easybuild.main import main_with_hooks -@click.command() -@EasyBuildCliOption.apply_options -@click.pass_context -@click.argument('other_args', nargs=-1, type=EasyconfigParam(), required=False) -def eb(ctx, other_args): - """EasyBuild command line interface.""" - args = [] - for key, value in getattr(ctx, 'hidden_params', {}).items(): - key = key.replace('_', '-') - opt = EasyBuildCliOption.OPTIONS_MAP[key] - # TEST_KEYS = [ - # 'robot', - # 'robot-paths', - # 'map-toolchains', - # 'search-paths', - # ] - # if key in TEST_KEYS: - # print(f'{key.upper()}: `{value=}` `{type(value)=}` `{opt.default=}` `{opt.action=}`') - if value in ['False', 'True']: - value = value == 'True' - if isinstance(value, bool): - if value != opt.default: - if value: - args.append(f"--{key}") - else: - args.append(f"--disable-{key}") - else: + @click.command() + @EasyBuildCliOption.apply_options + @click.pass_context + @click.argument('other_args', nargs=-1, type=EasyconfigParam(), required=False) + def eb(ctx, other_args): + """EasyBuild command line interface.""" + args = [] + for key, value in getattr(ctx, 'hidden_params', {}).items(): + key = key.replace('_', '-') + opt = EasyBuildCliOption.OPTIONS_MAP[key] + if value in ['False', 'True']: + value = value == 'True' + if isinstance(value, bool): + if value != opt.default: + if value: + args.append(f"--{key}") + else: + args.append(f"--disable-{key}") + else: - if isinstance(value, (list, tuple)) and value: - # Flatten nested lists if necessary - if isinstance(value[0], list): - value = sum(value, []) - # Match the type of the option with the default to see if we need to add it - if value and isinstance(value, list) and isinstance(opt.default, tuple): - value = tuple(value) - if value and isinstance(value, tuple) and isinstance(opt.default, list): - value = list(value) - value_is_default = (value == opt.default) + if isinstance(value, (list, tuple)) and value: + # Flatten nested lists if necessary + if isinstance(value[0], list): + value = sum(value, []) + # Match the type of the option with the default to see if we need to add it + if value and isinstance(value, list) and isinstance(opt.default, tuple): + value = tuple(value) + if value and isinstance(value, tuple) and isinstance(opt.default, list): + value = list(value) + value_is_default = (value == opt.default) - value_flattened = value - if isinstance(value, (list, tuple)): - if 'path' in opt.type: - delim = os.pathsep - elif 'str' in opt.type: - delim = ',' - elif 'url' in opt.type: - delim = '|' - else: - raise ValueError(f"Unsupported type for {key}: {opt.type}") - value_flattened = delim.join(value) + value_flattened = value + if isinstance(value, (list, tuple)): + if 'path' in opt.type: + delim = os.pathsep + elif 'str' in opt.type: + delim = ',' + elif 'url' in opt.type: + delim = '|' + else: + raise ValueError(f"Unsupported type for {key}: {opt.type}") + value_flattened = delim.join(value) - if opt.action == 'store_or_None': - if value is None or value == (): - continue - if value_is_default: - args.append(f"--{key}") - else: - args.append(f"--{key}={value_flattened}") - elif value and not value_is_default: - if value: - args.append(f"--{key}={value_flattened}") - else: - args.append(f"--{key}") - # for arg in args: - # print(f"ARG: {arg}") + if opt.action == 'store_or_None': + if value is None or value == (): + continue + if value_is_default: + args.append(f"--{key}") + else: + args.append(f"--{key}={value_flattened}") + elif value and not value_is_default: + if value: + args.append(f"--{key}={value_flattened}") + else: + args.append(f"--{key}") + for arg in args: + print(f"ARG: {arg}") - args.extend(other_args) - main_with_hooks(args=args) + args.extend(other_args) + main_with_hooks(args=args) From 9385b5288fd45046aa0fe552a6ed341fd72fa1c9 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 31 Jul 2025 13:56:43 +0200 Subject: [PATCH 13/33] Use pathsep as delimiter for paths and allow empty initial path in autocomplete --- easybuild/cli/options/__init__.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py index a1b7c18041..c67eae8e13 100644 --- a/easybuild/cli/options/__init__.py +++ b/easybuild/cli/options/__init__.py @@ -1,4 +1,3 @@ -# import logging import os from typing import Callable, Any @@ -55,7 +54,7 @@ def convert(self, value, param, ctx): return res def shell_complete(self, ctx, param, incomplete): - others, last = ([''] + incomplete.rsplit(self.delimiter, 1))[-2:] + others, last = ([None] + incomplete.rsplit(self.delimiter, 1))[-2:] # logging.warning(f"Shell completion for delimited path list: others={others}, last={last}") dir_path, prefix = os.path.split(last) dir_path = dir_path or '.' @@ -72,7 +71,7 @@ def shell_complete(self, ctx, param, incomplete): elif os.path.isfile(full_path): if self.file_okay: possibles.append(full_path) - start = f'{others}{self.delimiter}' if others else '' + start = f'{others}{self.delimiter}' if others is not None else '' res = [CompletionItem(f"{start}{path}") for path in possibles] # logging.warning(f"Shell completion for delimited path list: res={possibles}") return res @@ -105,10 +104,8 @@ class EasyconfigParam(click.ParamType): name = 'easyconfig' def shell_complete(self, ctx, param, incomplete): - if not incomplete: - return [] set_up_configuration(args=["--ignore-index"], silent=True, reconfigure=True) - return [CompletionItem(ec) for ec in search_easyconfigs(fr'^{incomplete}.*\.eb$', filename_only=True)] + return [CompletionItem(ec, help='') for ec in search_easyconfigs(fr'^{incomplete}.*\.eb$', filename_only=True)] @dataclass @@ -150,7 +147,7 @@ def to_click_option_dec(self): kwargs['type'] = DelimitedString(delimiter=',') kwargs['multiple'] = True elif self.type in ['pathlist', 'pathtuple']: - kwargs['type'] = DelimitedPathList(delimiter=',') + kwargs['type'] = DelimitedPathList(delimiter=os.pathsep) kwargs['multiple'] = True elif self.type in ['urllist', 'urltuple']: kwargs['type'] = DelimitedString(delimiter='|') From 50ba676c0f505e1f968413341447cf7a704b150a Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 31 Jul 2025 13:58:21 +0200 Subject: [PATCH 14/33] Passthrough the arguments from the CLI to optparse instead of rebuilding the, --- easybuild/cli/__init__.py | 61 +++------------------------------------ 1 file changed, 4 insertions(+), 57 deletions(-) diff --git a/easybuild/cli/__init__.py b/easybuild/cli/__init__.py index dd574adb1f..56770e7866 100644 --- a/easybuild/cli/__init__.py +++ b/easybuild/cli/__init__.py @@ -1,4 +1,3 @@ -import os import sys try: @@ -31,61 +30,9 @@ def eb(): @click.command() @EasyBuildCliOption.apply_options - @click.pass_context @click.argument('other_args', nargs=-1, type=EasyconfigParam(), required=False) - def eb(ctx, other_args): + def eb(other_args): """EasyBuild command line interface.""" - args = [] - for key, value in getattr(ctx, 'hidden_params', {}).items(): - key = key.replace('_', '-') - opt = EasyBuildCliOption.OPTIONS_MAP[key] - if value in ['False', 'True']: - value = value == 'True' - if isinstance(value, bool): - if value != opt.default: - if value: - args.append(f"--{key}") - else: - args.append(f"--disable-{key}") - else: - - if isinstance(value, (list, tuple)) and value: - # Flatten nested lists if necessary - if isinstance(value[0], list): - value = sum(value, []) - # Match the type of the option with the default to see if we need to add it - if value and isinstance(value, list) and isinstance(opt.default, tuple): - value = tuple(value) - if value and isinstance(value, tuple) and isinstance(opt.default, list): - value = list(value) - value_is_default = (value == opt.default) - - value_flattened = value - if isinstance(value, (list, tuple)): - if 'path' in opt.type: - delim = os.pathsep - elif 'str' in opt.type: - delim = ',' - elif 'url' in opt.type: - delim = '|' - else: - raise ValueError(f"Unsupported type for {key}: {opt.type}") - value_flattened = delim.join(value) - - if opt.action == 'store_or_None': - if value is None or value == (): - continue - if value_is_default: - args.append(f"--{key}") - else: - args.append(f"--{key}={value_flattened}") - elif value and not value_is_default: - if value: - args.append(f"--{key}={value_flattened}") - else: - args.append(f"--{key}") - for arg in args: - print(f"ARG: {arg}") - - args.extend(other_args) - main_with_hooks(args=args) + # Really no need to re-build the arguments if we support the exact same syntax we can just let them pass + # through to optparse + main_with_hooks() From 75fbc67f8b985610caf8133bd627483bac2f5066 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 31 Jul 2025 14:15:59 +0200 Subject: [PATCH 15/33] Improve autocomplete --- easybuild/cli/options/__init__.py | 39 ++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py index c67eae8e13..edba6d4ca6 100644 --- a/easybuild/cli/options/__init__.py +++ b/easybuild/cli/options/__init__.py @@ -18,6 +18,34 @@ opt_group.clear() # Clear existing groups to avoid conflicts +KNOWN_FILEPATH_OPTS = [ + 'hooks', + 'modules-footer', + 'modules-header', +] + +KNOWN_DIRPATH_OPTS = [ + 'locks-dir', + + 'failed-install-build-dirs-path', + 'failed-install-logs-path', + 'installpath-data', + 'installpath-modules', + 'installpath-software', + 'prefix', + 'sourcepath-data', + 'testoutput', + 'tmp-logdir', + 'tmpdir', + + 'buildpath', + 'containerpath', + 'installpath', + 'repositorypath', + 'sourcepath', +] + + class OptionExtracter(EasyBuildOptions): def __init__(self, *args, **kwargs): self._option_dicts = {} @@ -143,7 +171,14 @@ def to_click_option_dec(self): 'type': None } - if self.type in ['strlist', 'strtuple']: + # Manually enforced FILE types + if self.name in KNOWN_FILEPATH_OPTS: + kwargs['type'] = click.Path(exists=True, dir_okay=False, file_okay=True) + # Manually enforced DIRECTORY types + elif self.name in KNOWN_DIRPATH_OPTS: + kwargs['type'] = click.Path(exists=True, dir_okay=True, file_okay=False) + # Convert options from easybuild.tools.options + elif self.type in ['strlist', 'strtuple']: kwargs['type'] = DelimitedString(delimiter=',') kwargs['multiple'] = True elif self.type in ['pathlist', 'pathtuple']: @@ -162,6 +197,7 @@ def to_click_option_dec(self): kwargs['type'] = click.FLOAT elif self.type in ['str', str]: kwargs['type'] = click.STRING + # If type is None assume type based on default value elif self.type is None: if self.default is False or self.default is True: kwargs['is_flag'] = True @@ -172,6 +208,7 @@ def to_click_option_dec(self): kwargs['multiple'] = True kwargs['type'] = click.STRING + # store_or_None implies that the option can be used as a flag with no value if self.action == 'store_or_None': kwargs['default'] = None kwargs['flag_value'] = self.default From a015eaa042377e07bb049b9d5e6beb4c80643092 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 31 Jul 2025 14:36:33 +0200 Subject: [PATCH 16/33] Improve `help` metadata --- easybuild/cli/options/__init__.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py index edba6d4ca6..e31e3ebfd8 100644 --- a/easybuild/cli/options/__init__.py +++ b/easybuild/cli/options/__init__.py @@ -41,7 +41,6 @@ 'buildpath', 'containerpath', 'installpath', - 'repositorypath', 'sourcepath', ] @@ -63,13 +62,14 @@ class DelimitedPathList(click.Path): """Custom Click parameter type for delimited lists.""" name = 'pathlist' - def __init__(self, *args, delimiter=',', resolve_full: bool = False, **kwargs): + def __init__(self, *args, delimiter=',', **kwargs): + self.resolve_full = kwargs.setdefault('resolve_path', False) super().__init__(*args, **kwargs) self.delimiter = delimiter - self.resolve_full = resolve_full + name = self.name + self.name = f'[{name}[{self.delimiter}{name}]]' def convert(self, value, param, ctx): - # logging.warning(f"{param=} convert called with `{value=}`, `{type(value)=}`") if isinstance(value, str): res = value.split(self.delimiter) elif isinstance(value, (list, tuple)): @@ -78,12 +78,10 @@ def convert(self, value, param, ctx): raise click.BadParameter(f"Expected a comma-separated string, got {value}") if self.resolve_full: res = [os.path.abspath(v) for v in res] - # logging.warning(f"{param=} convert returning `{res=}`") return res def shell_complete(self, ctx, param, incomplete): others, last = ([None] + incomplete.rsplit(self.delimiter, 1))[-2:] - # logging.warning(f"Shell completion for delimited path list: others={others}, last={last}") dir_path, prefix = os.path.split(last) dir_path = dir_path or '.' # logging.warning(f"Shell completion for delimited path list: dir_path={dir_path}, prefix={prefix}") @@ -107,11 +105,10 @@ def shell_complete(self, ctx, param, incomplete): class DelimitedString(click.ParamType): """Custom Click parameter type for delimited strings.""" - name = 'strlist' - def __init__(self, *args, delimiter=',', **kwargs): super().__init__(*args, **kwargs) self.delimiter = delimiter + self.name = f'[STR[{self.delimiter}STR]]' def convert(self, value, param, ctx): if isinstance(value, str): @@ -173,20 +170,20 @@ def to_click_option_dec(self): # Manually enforced FILE types if self.name in KNOWN_FILEPATH_OPTS: - kwargs['type'] = click.Path(exists=True, dir_okay=False, file_okay=True) + kwargs['type'] = click.Path(dir_okay=False, file_okay=True) # Manually enforced DIRECTORY types elif self.name in KNOWN_DIRPATH_OPTS: - kwargs['type'] = click.Path(exists=True, dir_okay=True, file_okay=False) + kwargs['type'] = click.Path(dir_okay=True, file_okay=False) # Convert options from easybuild.tools.options elif self.type in ['strlist', 'strtuple']: kwargs['type'] = DelimitedString(delimiter=',') - kwargs['multiple'] = True + # kwargs['multiple'] = True elif self.type in ['pathlist', 'pathtuple']: kwargs['type'] = DelimitedPathList(delimiter=os.pathsep) - kwargs['multiple'] = True + # kwargs['multiple'] = True elif self.type in ['urllist', 'urltuple']: kwargs['type'] = DelimitedString(delimiter='|') - kwargs['multiple'] = True + # kwargs['multiple'] = True elif self.type == 'choice': if self.lst is None: raise ValueError(f"Choice type requires a list of choices for option {self.name}") @@ -202,7 +199,7 @@ def to_click_option_dec(self): if self.default is False or self.default is True: kwargs['is_flag'] = True kwargs['type'] = click.BOOL - if self.action in ['store_true', 'store_false']: + if self.default is True: decl = f"--{self.name}/--disable-{self.name}" elif isinstance(self.default, (list, tuple)): kwargs['multiple'] = True From d2dd8b2f7e2e7863517bff66290a2c8d92ca89fc Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 31 Jul 2025 14:43:16 +0200 Subject: [PATCH 17/33] Lint and better comments --- easybuild/cli/__init__.py | 7 ++++--- easybuild/main.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/cli/__init__.py b/easybuild/cli/__init__.py index 56770e7866..322ee1ec73 100644 --- a/easybuild/cli/__init__.py +++ b/easybuild/cli/__init__.py @@ -5,9 +5,10 @@ except ImportError: def eb(): """Placeholder function to inform the user that `click` is required.""" - print("Using `eb2` requires `click` to be installed. Either use `eb` or install `click` with `pip install click`.") - print("`eb2` also uses `rich` and `rich_click` as optional dependencies for enhanced CLI experience.") - print("Exiting...") + print('Using `eb2` requires `click` to be installed.') + print('Either use `eb` or install `click` with `pip install click`.') + print('`eb2` also uses `rich` and `rich_click` as optional dependencies for enhanced CLI experience.') + print('Exiting...') sys.exit(0) else: try: diff --git a/easybuild/main.py b/easybuild/main.py index 00ea3a19a2..b37e66e387 100755 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -838,6 +838,7 @@ def main_with_hooks(args=None): except EasyBuildError as err: print_error(err.msg, exit_code=err.exit_code) else: + # Avoid running double initialization in `main` afterward if `prepare_main` succeeded args = None hooks = load_hooks(eb_go.options.hooks) From 755d496a37884caf606e3a6ba954581412d4f1b8 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 31 Jul 2025 15:05:00 +0200 Subject: [PATCH 18/33] Added optional dependencies `eb2` to install packages required by `eb2` --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 0c29403c69..e79744ffaa 100644 --- a/setup.py +++ b/setup.py @@ -124,6 +124,9 @@ def find_rel_test(): 'eb2 = easybuild.cli:eb', ] }, + extras_require={ + 'eb2': ['click', 'rich', 'rich_click'], + }, data_files=[ ('easybuild/scripts', glob.glob('easybuild/scripts/*')), ('etc', glob.glob('etc/*')), From f04fe9ed77e719d2754536880a0af0d16830b884 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 12 Feb 2026 14:53:20 +0100 Subject: [PATCH 19/33] Make the click based CLI opt-in only if you have click installed --- easybuild/cli/__init__.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/easybuild/cli/__init__.py b/easybuild/cli/__init__.py index 322ee1ec73..c57a4da9c0 100644 --- a/easybuild/cli/__init__.py +++ b/easybuild/cli/__init__.py @@ -1,15 +1,16 @@ -import sys +from easybuild.main import main_with_hooks try: import click as original_click except ImportError: def eb(): """Placeholder function to inform the user that `click` is required.""" - print('Using `eb2` requires `click` to be installed.') - print('Either use `eb` or install `click` with `pip install click`.') - print('`eb2` also uses `rich` and `rich_click` as optional dependencies for enhanced CLI experience.') - print('Exiting...') - sys.exit(0) + main_with_hooks() + # print('Using `eb2` requires `click` to be installed.') + # print('Either use `eb` or install `click` with `pip install click`.') + # print('`eb2` also uses `rich` and `rich_click` as optional dependencies for enhanced CLI experience.') + # print('Exiting...') + # sys.exit(0) else: try: import rich_click as click @@ -27,8 +28,6 @@ def eb(): from .options import EasyBuildCliOption, EasyconfigParam - from easybuild.main import main_with_hooks - @click.command() @EasyBuildCliOption.apply_options @click.argument('other_args', nargs=-1, type=EasyconfigParam(), required=False) From cf8bbb2e73d5468553ad83a53a52f9de8cbd6cdb Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 17 Feb 2026 14:13:42 +0100 Subject: [PATCH 20/33] Add CLI packages to `setup.py` --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e79744ffaa..5ba38dd381 100644 --- a/setup.py +++ b/setup.py @@ -77,6 +77,7 @@ def find_rel_test(): "easybuild.tools.module_naming_scheme", "easybuild.tools.package", "easybuild.tools.package.package_naming_scheme", "easybuild.tools.py2vs3", "easybuild.tools.repository", "easybuild.tools.tomllib", "easybuild.tools.tomllib.tomli", "easybuild.tools._toml_writer", + "easybuild.cli", "easybuild.cli.options", "test.framework", "test", ] From 1764d2f7841b79d9c07c6c0bdebc014922e85c35 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 20 Feb 2026 10:44:42 +0100 Subject: [PATCH 21/33] Improvements to pattern matching and removed unused --- easybuild/cli/options/__init__.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py index e31e3ebfd8..71020ff7c9 100644 --- a/easybuild/cli/options/__init__.py +++ b/easybuild/cli/options/__init__.py @@ -1,4 +1,5 @@ import os +import re from typing import Callable, Any from dataclasses import dataclass @@ -97,7 +98,12 @@ def shell_complete(self, ctx, param, incomplete): elif os.path.isfile(full_path): if self.file_okay: possibles.append(full_path) - start = f'{others}{self.delimiter}' if others is not None else '' + if others is None: + start = '' + elif others == '': + start = self.delimiter + else: + start = f'{others}{self.delimiter}' res = [CompletionItem(f"{start}{path}") for path in possibles] # logging.warning(f"Shell completion for delimited path list: res={possibles}") return res @@ -129,8 +135,14 @@ class EasyconfigParam(click.ParamType): name = 'easyconfig' def shell_complete(self, ctx, param, incomplete): + incomplete = re.escape(incomplete) set_up_configuration(args=["--ignore-index"], silent=True, reconfigure=True) - return [CompletionItem(ec, help='') for ec in search_easyconfigs(fr'^{incomplete}.*\.eb$', filename_only=True)] + return [ + CompletionItem(ec, help='') for ec in search_easyconfigs( + fr'^(?={incomplete}).*\.eb$', + filename_only=True + ) + ] @dataclass @@ -188,6 +200,8 @@ def to_click_option_dec(self): if self.lst is None: raise ValueError(f"Choice type requires a list of choices for option {self.name}") kwargs['type'] = click.Choice(self.lst, case_sensitive=True) + if self.default is not None: + kwargs['is_flag'] = True elif self.type in ['int', int]: kwargs['type'] = click.INT elif self.type in ['float', float]: @@ -215,17 +229,9 @@ def to_click_option_dec(self): return click.option( *decls, expose_value=False, - callback=self.register_hidden_param, **kwargs ) - @staticmethod - def register_hidden_param(ctx, param, value): - """Register a hidden parameter in the context.""" - if not hasattr(ctx, 'hidden_params'): - ctx.hidden_params = {} - ctx.hidden_params[param.name] = value - class EasyBuildCliOption(): OPTIONS: list[OptionData] = [] From 808a1730b5eba644eed6e471d2918acf8ffc3e34 Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 11 Mar 2026 15:47:18 +0100 Subject: [PATCH 22/33] Replaces standard EB cli with click-wrapped one + ensures that autocompletion script can still be generated --- easybuild/cli/__main__.py | 4 ++++ eb | 2 +- setup.py | 5 ----- 3 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 easybuild/cli/__main__.py diff --git a/easybuild/cli/__main__.py b/easybuild/cli/__main__.py new file mode 100644 index 0000000000..54ce89775e --- /dev/null +++ b/easybuild/cli/__main__.py @@ -0,0 +1,4 @@ +from easybuild.cli import eb + +# Ensure Click to recognizes the program name as `eb` when invoked as `python -m easybuild.cli` or similar +eb(prog_name='eb') diff --git a/eb b/eb index 132cf92ecf..0a411fc02e 100755 --- a/eb +++ b/eb @@ -44,7 +44,7 @@ trap keyboard_interrupt SIGINT # Python 3.6+ required REQ_MIN_PY3VER=6 -EASYBUILD_MAIN='easybuild.main' +EASYBUILD_MAIN='easybuild.cli' # easybuild module to import to check whether EasyBuild framework is available; # don't use easybuild.main here, since that's a very expensive module to import (it makes the 'eb' command slow) diff --git a/setup.py b/setup.py index 5ba38dd381..61741ecffa 100644 --- a/setup.py +++ b/setup.py @@ -120,11 +120,6 @@ def find_rel_test(): # utility scripts 'easybuild/scripts/install_eb_dep.sh', ], - entry_points={ - 'console_scripts': [ - 'eb2 = easybuild.cli:eb', - ] - }, extras_require={ 'eb2': ['click', 'rich', 'rich_click'], }, From 5439684362e867e9d78de889db1ac4aca2a52ca5 Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 11 Mar 2026 15:55:26 +0100 Subject: [PATCH 23/33] Fix args for non-click shim --- easybuild/cli/__init__.py | 7 +------ setup.py | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/easybuild/cli/__init__.py b/easybuild/cli/__init__.py index c57a4da9c0..770ba32feb 100644 --- a/easybuild/cli/__init__.py +++ b/easybuild/cli/__init__.py @@ -3,14 +3,9 @@ try: import click as original_click except ImportError: - def eb(): + def eb(*args, **kwargs): """Placeholder function to inform the user that `click` is required.""" main_with_hooks() - # print('Using `eb2` requires `click` to be installed.') - # print('Either use `eb` or install `click` with `pip install click`.') - # print('`eb2` also uses `rich` and `rich_click` as optional dependencies for enhanced CLI experience.') - # print('Exiting...') - # sys.exit(0) else: try: import rich_click as click diff --git a/setup.py b/setup.py index 61741ecffa..c5f1a9369c 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,7 @@ def find_rel_test(): 'easybuild/scripts/install_eb_dep.sh', ], extras_require={ - 'eb2': ['click', 'rich', 'rich_click'], + 'eb_click': ['click', 'rich', 'rich_click'], }, data_files=[ ('easybuild/scripts', glob.glob('easybuild/scripts/*')), From 5872704be7b109d99721e0e7600d9948c6265ee6 Mon Sep 17 00:00:00 2001 From: Davide Grassano <34096612+Crivella@users.noreply.github.com> Date: Thu, 9 Apr 2026 10:35:29 +0200 Subject: [PATCH 24/33] Apply suggestion from @boegel Co-authored-by: Kenneth Hoste --- eb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eb b/eb index 0a411fc02e..c5cba9b1a4 100755 --- a/eb +++ b/eb @@ -44,7 +44,12 @@ trap keyboard_interrupt SIGINT # Python 3.6+ required REQ_MIN_PY3VER=6 -EASYBUILD_MAIN='easybuild.cli' +if [ "${EB_CLI_CLICK}" == "1" ]; then + # opt-in to using Click-based CLI (if click is available) + EASYBUILD_MAIN='easybuild.cli' +else + EASYBUILD_MAIN='easybuild.main' +fi # easybuild module to import to check whether EasyBuild framework is available; # don't use easybuild.main here, since that's a very expensive module to import (it makes the 'eb' command slow) From cc0f0d9f18830c96f924ee2ed4a92a07e3c4c0c7 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 9 Apr 2026 10:45:06 +0200 Subject: [PATCH 25/33] Add copyright/license headers to new files --- easybuild/cli/__init__.py | 24 ++++++++++++++++++++++++ easybuild/cli/__main__.py | 24 ++++++++++++++++++++++++ easybuild/cli/options/__init__.py | 24 ++++++++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/easybuild/cli/__init__.py b/easybuild/cli/__init__.py index 770ba32feb..905b3d7b07 100644 --- a/easybuild/cli/__init__.py +++ b/easybuild/cli/__init__.py @@ -1,3 +1,27 @@ +# # +# Copyright 2009-2026 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # from easybuild.main import main_with_hooks try: diff --git a/easybuild/cli/__main__.py b/easybuild/cli/__main__.py index 54ce89775e..20976142b2 100644 --- a/easybuild/cli/__main__.py +++ b/easybuild/cli/__main__.py @@ -1,3 +1,27 @@ +# # +# Copyright 2009-2026 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # from easybuild.cli import eb # Ensure Click to recognizes the program name as `eb` when invoked as `python -m easybuild.cli` or similar diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py index 71020ff7c9..eb908bc769 100644 --- a/easybuild/cli/options/__init__.py +++ b/easybuild/cli/options/__init__.py @@ -1,3 +1,27 @@ +# # +# Copyright 2009-2026 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# # import os import re From e1120f055eb6bad842c97214a9b52189f9613798 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 9 Apr 2026 11:10:37 +0200 Subject: [PATCH 26/33] Add version option to the click CLI --- easybuild/cli/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/cli/__init__.py b/easybuild/cli/__init__.py index 905b3d7b07..cc14040833 100644 --- a/easybuild/cli/__init__.py +++ b/easybuild/cli/__init__.py @@ -46,10 +46,12 @@ def eb(*args, **kwargs): ]) from .options import EasyBuildCliOption, EasyconfigParam + from easybuild.tools.version import this_is_easybuild @click.command() @EasyBuildCliOption.apply_options @click.argument('other_args', nargs=-1, type=EasyconfigParam(), required=False) + @click.version_option(version=this_is_easybuild(), message='%(version)s') def eb(other_args): """EasyBuild command line interface.""" # Really no need to re-build the arguments if we support the exact same syntax we can just let them pass From d858e66b207e1b1e404b844e1ddee656a9858bf2 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 9 Apr 2026 11:34:02 +0200 Subject: [PATCH 27/33] Add end2end test on ubuntu 24 using the new click CLI --- .github/workflows/end2end.yml | 45 +++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 3954a3ad7f..e9e78166b5 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -64,3 +64,48 @@ jobs: EB_ARGS='--filter-deps=binutils' fi sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot ${EB_ARGS}" + + - name: Create a virtual environment and install EasyBuild with the click cli extra + if: matrix.container == 'ubuntu-24.04' + shell: bash + run: | + apt update && apt install -y python3-venv + python3 -m venv venv + source venv/bin/activate + pip install --upgrade pip + pip install .[eb_click] + pip install $HOME/easybuild-easyblocks-develop + pip install $HOME/easybuild-easyconfigs-develop + chown -R easybuild:easybuild venv + chmod 755 /root + + - name: Run commands to check test environment using the click cli + if: matrix.container == 'ubuntu-24.04' + shell: bash + run: | + cmds=( + "whoami" + "pwd" + "env | sort" + "eb --help" + "eb --version" + "eb --help | grep '─ Options ─'" + "eb --show-system-info" + "eb --check-eb-deps" + "eb --show-config" + "eb -x bzip2-1.0.8.eb" + ) + for cmd in "${cmds[@]}"; do + echo ">>> $cmd" + sudo -u easybuild bash -l -c "export EB_CLI_CLICK=1; source /root/venv/bin/activate; $cmd" + done + + - name: End-to-end test of installing bzip2 with EasyBuild using the click cli + if: matrix.container == 'ubuntu-24.04' + shell: bash + run: | + EB_ARGS='' + if [[ "${{ matrix.container }}" == "fedora-41" ]] || [[ "${{ matrix.container }}" == "ubuntu-24.04" ]]; then + EB_ARGS='--filter-deps=binutils' + fi + sudo -u easybuild bash -l -c "export EB_CLI_CLICK=1; source /root/venv/bin/activate; eb bzip2-1.0.8.eb --trace --robot ${EB_ARGS}" From 0267a3c59be2d3ad6924c9ce0db38ba8d52fe65d Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 9 Apr 2026 12:50:16 +0200 Subject: [PATCH 28/33] Get correct VENV directory --- .github/workflows/end2end.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index e9e78166b5..d56f028303 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -69,20 +69,21 @@ jobs: if: matrix.container == 'ubuntu-24.04' shell: bash run: | + export VENVDIR=/home/easybuild/venv apt update && apt install -y python3-venv - python3 -m venv venv - source venv/bin/activate + python3 -m venv $VENVDIR + source $VENVDIR/bin/activate pip install --upgrade pip pip install .[eb_click] pip install $HOME/easybuild-easyblocks-develop pip install $HOME/easybuild-easyconfigs-develop - chown -R easybuild:easybuild venv - chmod 755 /root + chown -R easybuild:easybuild $VENVDIR - name: Run commands to check test environment using the click cli if: matrix.container == 'ubuntu-24.04' shell: bash run: | + export VENVDIR=/home/easybuild/venv cmds=( "whoami" "pwd" @@ -97,15 +98,16 @@ jobs: ) for cmd in "${cmds[@]}"; do echo ">>> $cmd" - sudo -u easybuild bash -l -c "export EB_CLI_CLICK=1; source /root/venv/bin/activate; $cmd" + sudo -u easybuild bash -l -c "export EB_CLI_CLICK=1; source $VENVDIR/bin/activate; $cmd" done - name: End-to-end test of installing bzip2 with EasyBuild using the click cli if: matrix.container == 'ubuntu-24.04' shell: bash run: | + export VENVDIR=/home/easybuild/venv EB_ARGS='' if [[ "${{ matrix.container }}" == "fedora-41" ]] || [[ "${{ matrix.container }}" == "ubuntu-24.04" ]]; then EB_ARGS='--filter-deps=binutils' fi - sudo -u easybuild bash -l -c "export EB_CLI_CLICK=1; source /root/venv/bin/activate; eb bzip2-1.0.8.eb --trace --robot ${EB_ARGS}" + sudo -u easybuild bash -l -c "export EB_CLI_CLICK=1; source $VENVDIR/bin/activate; eb bzip2-1.0.8.eb --trace --robot ${EB_ARGS}" From bfc24d8e07aaa624b8b5a4e12f07c4ca3a46889b Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 9 Apr 2026 13:17:37 +0200 Subject: [PATCH 29/33] Run new CLI test on all end2end containers --- .github/workflows/end2end.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index d56f028303..f7b5d303f2 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -65,12 +65,16 @@ jobs: fi sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot ${EB_ARGS}" + - name: Ensure `python3 -m venv` can be used + if: startsWith(matrix.container, 'ubuntu') + shell: bash + run: | + apt update && apt install -y python3-venv + - name: Create a virtual environment and install EasyBuild with the click cli extra - if: matrix.container == 'ubuntu-24.04' shell: bash run: | export VENVDIR=/home/easybuild/venv - apt update && apt install -y python3-venv python3 -m venv $VENVDIR source $VENVDIR/bin/activate pip install --upgrade pip @@ -80,7 +84,6 @@ jobs: chown -R easybuild:easybuild $VENVDIR - name: Run commands to check test environment using the click cli - if: matrix.container == 'ubuntu-24.04' shell: bash run: | export VENVDIR=/home/easybuild/venv @@ -102,7 +105,6 @@ jobs: done - name: End-to-end test of installing bzip2 with EasyBuild using the click cli - if: matrix.container == 'ubuntu-24.04' shell: bash run: | export VENVDIR=/home/easybuild/venv From fc21888db442b6d6f5cc3717856c3a9cf3878e6b Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 9 Apr 2026 13:42:57 +0200 Subject: [PATCH 30/33] Make old python happy with typehints --- easybuild/cli/options/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py index eb908bc769..d36b0cb62d 100644 --- a/easybuild/cli/options/__init__.py +++ b/easybuild/cli/options/__init__.py @@ -25,7 +25,7 @@ import os import re -from typing import Callable, Any +from typing import Callable, Any, List, Dict from dataclasses import dataclass from click.shell_completion import CompletionItem @@ -178,8 +178,8 @@ class OptionData: default: Any group: str = None short: str = None - meta: dict = None - lst: list = None + meta: Dict = None + lst: List = None def __post_init__(self): if self.short is not None and not isinstance(self.short, str): @@ -258,8 +258,8 @@ def to_click_option_dec(self): class EasyBuildCliOption(): - OPTIONS: list[OptionData] = [] - OPTIONS_MAP: dict[str, OptionData] = {} + OPTIONS: List[OptionData] = [] + OPTIONS_MAP: Dict[str, OptionData] = {} @classmethod def apply_options(cls, function: Callable) -> Callable: From 42a15362bf29017bbeda0b9f694045560d766889 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 9 Apr 2026 15:20:41 +0200 Subject: [PATCH 31/33] choices should probably not be a flag by default unless `store_or_XXX` is used. Allow list values to be False/None for `store_or_XXX` --- easybuild/cli/options/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/easybuild/cli/options/__init__.py b/easybuild/cli/options/__init__.py index d36b0cb62d..921968e198 100644 --- a/easybuild/cli/options/__init__.py +++ b/easybuild/cli/options/__init__.py @@ -99,6 +99,8 @@ def convert(self, value, param, ctx): res = value.split(self.delimiter) elif isinstance(value, (list, tuple)): res = value + elif value is False or value is None: + res = value else: raise click.BadParameter(f"Expected a comma-separated string, got {value}") if self.resolve_full: @@ -145,6 +147,8 @@ def convert(self, value, param, ctx): res = value.split(self.delimiter) elif isinstance(value, (list, tuple)): res = value + elif value is False or value is None: + res = value else: raise click.BadParameter(f"Expected a string or a comma-separated string, got {value}") return res @@ -224,8 +228,8 @@ def to_click_option_dec(self): if self.lst is None: raise ValueError(f"Choice type requires a list of choices for option {self.name}") kwargs['type'] = click.Choice(self.lst, case_sensitive=True) - if self.default is not None: - kwargs['is_flag'] = True + # if self.default is not None: + # kwargs['is_flag'] = True elif self.type in ['int', int]: kwargs['type'] = click.INT elif self.type in ['float', float]: @@ -247,6 +251,9 @@ def to_click_option_dec(self): if self.action == 'store_or_None': kwargs['default'] = None kwargs['flag_value'] = self.default + elif self.action == 'store_or_False': + kwargs['default'] = False + kwargs['flag_value'] = self.default decls = other_decls + [decl] @@ -326,4 +333,5 @@ def register_option(cls, group: str, name: str, data: tuple, prefix: str = '') - if dct is None: continue for key, value in dct.items(): + # print(f"Registering option: group={grp}, key={key}, value={value}, prefix={prefix}") EasyBuildCliOption.register_option(grp, key, value, prefix=prefix) From 86b0ab314acf812bb0d0cca87a569939a0e52d73 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 9 Apr 2026 17:23:32 +0200 Subject: [PATCH 32/33] Only chown user as group might not exist --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index f7b5d303f2..19774fa913 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -81,7 +81,7 @@ jobs: pip install .[eb_click] pip install $HOME/easybuild-easyblocks-develop pip install $HOME/easybuild-easyconfigs-develop - chown -R easybuild:easybuild $VENVDIR + chown -R easybuild $VENVDIR - name: Run commands to check test environment using the click cli shell: bash From 5413db8c983a8ede5afb98697f9cc040c06ef2ab Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 10 Apr 2026 10:13:49 +0200 Subject: [PATCH 33/33] Add `--rebuild` to the end2end test with click --- .github/workflows/end2end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml index 19774fa913..bb016ca5e7 100644 --- a/.github/workflows/end2end.yml +++ b/.github/workflows/end2end.yml @@ -112,4 +112,4 @@ jobs: if [[ "${{ matrix.container }}" == "fedora-41" ]] || [[ "${{ matrix.container }}" == "ubuntu-24.04" ]]; then EB_ARGS='--filter-deps=binutils' fi - sudo -u easybuild bash -l -c "export EB_CLI_CLICK=1; source $VENVDIR/bin/activate; eb bzip2-1.0.8.eb --trace --robot ${EB_ARGS}" + sudo -u easybuild bash -l -c "export EB_CLI_CLICK=1; source $VENVDIR/bin/activate; eb bzip2-1.0.8.eb --rebuild --trace --robot ${EB_ARGS}"