From 3ea5281c2ce1ec27a2f2967a9155f9f021820069 Mon Sep 17 00:00:00 2001 From: Xavier Besseron Date: Thu, 23 Apr 2026 15:24:50 +0200 Subject: [PATCH 1/2] Fix typo --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 63c923581f..599558a51e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5242,7 +5242,7 @@ def test_github_empty_pr(self): self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, do_build=True, raise_error=True) def test_show_config(self): - """"Test --show-config and --show-full-config.""" + """Test --show-config and --show-full-config.""" # only retain $EASYBUILD_* environment variables we expect for this test retained_eb_env_vars = [ From 189c29602c7d66544e96183a3f32a80da2dc4598 Mon Sep 17 00:00:00 2001 From: Xavier Besseron Date: Thu, 23 Apr 2026 15:26:02 +0200 Subject: [PATCH 2/2] Add option --include-job-backend --- easybuild/tools/include.py | 32 ++++++++++++++++++++ easybuild/tools/job/__init__.py | 32 ++++++++++++++++++++ easybuild/tools/options.py | 14 ++++++++- test/framework/include.py | 42 +++++++++++++++++++++++++- test/framework/options.py | 52 +++++++++++++++++++++++++++++++++ 5 files changed, 170 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 6180731f01..9c026647df 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -293,3 +293,35 @@ def include_toolchains(tmpdir, paths): verify_imports([os.path.splitext(tcmod)[0] for tcmod in included_subpkg_modules[subpkg]], pkg, loc) return toolchains_path + + +def include_job_backends(tmpdir, paths): + """Include job backends at specified locations.""" + job_backends_path = os.path.join(tmpdir, 'included-job-backends') + + set_up_eb_package(job_backends_path, 'easybuild.tools.job') + + job_backends_dir = os.path.join(job_backends_path, 'easybuild', 'tools', 'job') + + allpaths = [p for p in expand_glob_paths(paths) if os.path.basename(p) != '__init__.py'] + for job_backend in allpaths: + filename = os.path.basename(job_backend) + target_path = os.path.join(job_backends_dir, filename) + if not os.path.exists(target_path): + symlink(job_backend, target_path) + + included_job_backends = [x for x in os.listdir(job_backends_dir) if x not in ['__init__.py']] + _log.debug("Included job backends: %s", included_job_backends) + + # inject path into Python search path, and reload modules to get it 'registered' in sys.modules + sys.path.insert(0, job_backends_path) + + # hard inject location to included module naming schemes into Python search path + # only prepending to sys.path is not enough due to 'pkgutil.extend_path' in job/__init__.py + new_path = os.path.join(job_backends_path, 'easybuild', 'tools', 'job') + easybuild.tools.job.__path__.insert(0, new_path) + + # sanity check: verify that included job backends can be imported (from expected location) + verify_imports([os.path.splitext(mns)[0] for mns in included_job_backends], 'easybuild.tools.job', job_backends_dir) + + return job_backends_path diff --git a/easybuild/tools/job/__init__.py b/easybuild/tools/job/__init__.py index e69de29bb2..8221a7ff04 100644 --- a/easybuild/tools/job/__init__.py +++ b/easybuild/tools/job/__init__.py @@ -0,0 +1,32 @@ +## +# Copyright 2011-2026 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Declares easybuild.tools.job namespace, in an extendable way. + +Authors: + +* Xavier Besseron (LuxProvide) +""" +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 96bb8a484a..4470d54f91 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -93,7 +93,8 @@ from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS from easybuild.tools.github import fetch_easyblocks_from_commit, fetch_easyblocks_from_pr, fetch_github_token from easybuild.tools.hooks import KNOWN_HOOKS -from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains +from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains, \ + include_job_backends from easybuild.tools.job.backend import avail_job_backends from easybuild.tools.modules import avail_modules_tools from easybuild.tools.module_generator import ModuleGeneratorLua, avail_module_generators @@ -640,6 +641,8 @@ def config_options(self): opts = OrderedDict({ 'avail-module-naming-schemes': ("Show all supported module naming schemes", None, 'store_true', False,), + 'avail-job-backends': ("Show all supported job backends", + None, "store_true", False,), 'avail-modules-tools': ("Show all supported module tools", None, "store_true", False,), 'avail-repositories': ("Show all repository types (incl. non-usable)", @@ -668,6 +671,7 @@ def config_options(self): 'strlist', 'store', []), 'include-toolchains': ("Location(s) of extra or customized toolchains or toolchain components", 'strlist', 'store', []), + 'include-job-backends': ("Location(s) of extra or customized job backends", 'strlist', 'store', []), 'installpath': ("Install path for software and modules", None, 'store', mk_full_default_path('installpath')), 'installpath-data': ("Install path for data (if None, combine --installpath and --subdir-data)", @@ -1078,6 +1082,7 @@ def postprocess(self): self.options.avail_modules_tools, self.options.avail_module_naming_schemes, self.options.show_default_configfiles, self.options.avail_toolchain_opts, self.options.avail_hooks, self.options.show_system_info, + self.options.avail_job_backends )): build_easyconfig_constants_dict() # runs the easyconfig constants sanity check self._postprocess_list_avail() @@ -1208,6 +1213,9 @@ def _postprocess_include(self): if self.options.include_toolchains: include_toolchains(self.tmpdir, self.options.include_toolchains) + if self.options.include_job_backends: + include_job_backends(self.tmpdir, self.options.include_job_backends) + def _postprocess_checks(self): """Check whether (combination of) configuration options make sense.""" @@ -1459,6 +1467,10 @@ def _postprocess_list_avail(self): if self.options.avail_module_naming_schemes: msg += self.avail_list('module naming schemes', avail_module_naming_schemes()) + # dump supported job backends + if self.options.avail_job_backends: + msg += self.avail_list('job backends', avail_job_backends()) + # dump default list of config files that are considered if self.options.show_default_configfiles: msg += self.show_default_configfiles() diff --git a/test/framework/include.py b/test/framework/include.py index 9808dc85ee..9c9a813aa0 100644 --- a/test/framework/include.py +++ b/test/framework/include.py @@ -34,7 +34,8 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import mkdir, write_file -from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains +from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains, \ + include_job_backends from easybuild.tools.include import is_software_specific_easyblock @@ -243,6 +244,45 @@ def test_include_toolchains(self): my_tc_real_py_path = os.path.realpath(os.path.join(os.path.dirname(my_tc_pyc_path), 'my_tc.py')) self.assertTrue(os.path.samefile(up(my_tc_real_py_path, 1), my_toolchains)) + def test_include_job_backend(self): + """Test include_job_backends().""" + + my_job_backend = os.path.join(self.test_prefix, 'my_job_backend') + mkdir(my_job_backend) + + # include __init__.py file that should be ignored, and shouldn't cause trouble (bug #1697) + write_file(os.path.join(my_job_backend, '__init__.py'), "# dummy init, should not get included") + + my_job_backend_txt = '\n'.join([ + "from easybuild.tools.job.backend import JobBackend", + "class MyJobBackend(JobBackend):", + " pass", + ]) + write_file(os.path.join(my_job_backend, 'my_job_backend.py'), my_job_backend_txt) + + my_job_backend_bis = os.path.join(self.test_prefix, 'my_job_backend.py') + write_file(my_job_backend_bis, '') + + # include custom job backend + included_job_backends_path = include_job_backends(self.test_prefix, [os.path.join(my_job_backend, '*.py'), + my_job_backend_bis]) + + expected_paths = ['__init__.py', 'tools/__init__.py', 'tools/job/__init__.py', + 'tools/job/my_job_backend.py'] + for filepath in expected_paths: + fullpath = os.path.join(included_job_backends_path, 'easybuild', filepath) + self.assertExists(fullpath) + + # path to included job backends should be prepended to Python search path + self.assertEqual(sys.path[0], included_job_backends_path) + + # importing custom job backends should work + import easybuild.tools.job.my_job_backend + my_job_backend_pyc_path = easybuild.tools.job.my_job_backend.__file__ + my_job_backend_real_py_path = os.path.realpath(os.path.join(os.path.dirname(my_job_backend_pyc_path), + 'my_job_backend.py')) + self.assertTrue(os.path.samefile(up(my_job_backend_real_py_path, 1), my_job_backend)) + def test_is_software_specific_easyblock(self): """Test is_software_specific_easyblock function.""" diff --git a/test/framework/options.py b/test/framework/options.py index 599558a51e..7bdbc921e8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4183,6 +4183,58 @@ def test_include_toolchains(self): res = run_shell_cmd(test_cmd) self.assertRegex(res.output, tc_regex) + def test_include_job_backends(self): + """Test --include-job-backends.""" + + # make sure that calling out to 'eb' will work by restoring $PATH & $PYTHONPATH + self.restore_env_path_pythonpath() + + topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + # try and make sure 'eb' is available via $PATH if it isn't yet + path = self.env_path + if which('eb') is None: + path = '%s:%s' % (topdir, path) + + # try and make sure top-level directory is in $PYTHONPATH if it isn't yet + pythonpath = self.env_pythonpath + with self.mocked_stdout_stderr(): + res = run_shell_cmd("cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False) + if res.exit_code != 0: + pythonpath = '%s:%s' % (topdir, pythonpath) + + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # clear log + write_file(self.logfile, '') + + job_backend_regex = re.compile(r'^\s*TestIncludedJobBackend', re.M) + + # TestIncludedJobBackend job backend is not available by default + args = ['--avail-job-backends'] + test_cmd = self.mk_eb_test_cmd(args) + with self.mocked_stdout_stderr(): + res = run_shell_cmd(test_cmd) + self.assertNotRegex(res.output, job_backend_regex) + + # include extra test job backend + job_backend_txt = '\n'.join([ + 'from easybuild.tools.job.backend import JobBackend', + 'class TestIncludedJobBackend(JobBackend):', + ' pass', + ]) + write_file(os.path.join(self.test_prefix, 'test_job_backend.py'), job_backend_txt) + + # clear log + write_file(self.logfile, '') + + args.append('--include-job-backends=%s/*.py' % self.test_prefix) + test_cmd = self.mk_eb_test_cmd(args) + with self.mocked_stdout_stderr(): + res = run_shell_cmd(test_cmd) + self.assertRegex(res.output, job_backend_regex) + def test_cleanup_tmpdir(self): """Test --cleanup-tmpdir.""" topdir = os.path.dirname(os.path.abspath(__file__))