Add options --include-job-backends and --avail-job-backends

easybuild/tools/include.py

@@ -293,3 +293,35 @@ def include_toolchains(tmpdir, paths):
         verify_imports([os.path.splitext(tcmod)[0] for tcmod in included_subpkg_modules[subpkg]], pkg, loc)
 
     return toolchains_path
+
+
+def include_job_backends(tmpdir, paths):
+    """Include job backends at specified locations."""
+    job_backends_path = os.path.join(tmpdir, 'included-job-backends')
+
+    set_up_eb_package(job_backends_path, 'easybuild.tools.job')
+
+    job_backends_dir = os.path.join(job_backends_path, 'easybuild', 'tools', 'job')
+
+    allpaths = [p for p in expand_glob_paths(paths) if os.path.basename(p) != '__init__.py']
+    for job_backend in allpaths:
+        filename = os.path.basename(job_backend)
+        target_path = os.path.join(job_backends_dir, filename)
+        if not os.path.exists(target_path):
+            symlink(job_backend, target_path)
+
+    included_job_backends = [x for x in os.listdir(job_backends_dir) if x not in ['__init__.py']]
+    _log.debug("Included job backends: %s", included_job_backends)
+
+    # inject path into Python search path, and reload modules to get it 'registered' in sys.modules
+    sys.path.insert(0, job_backends_path)
+
+    # hard inject location to included module naming schemes into Python search path
+    # only prepending to sys.path is not enough due to 'pkgutil.extend_path' in job/__init__.py
+    new_path = os.path.join(job_backends_path, 'easybuild', 'tools', 'job')
+    easybuild.tools.job.__path__.insert(0, new_path)
+
+    # sanity check: verify that included job backends can be imported (from expected location)
+    verify_imports([os.path.splitext(mns)[0] for mns in included_job_backends], 'easybuild.tools.job', job_backends_dir)
+
+    return job_backends_path

easybuild/tools/job/__init__.py

@@ -0,0 +1,32 @@
+##
+# Copyright 2011-2026 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+Declares easybuild.tools.job namespace, in an extendable way.
+
+Authors:
+
+* Xavier Besseron (LuxProvide)
+"""
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

easybuild/tools/options.py

@@ -93,7 +93,8 @@
 from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS
 from easybuild.tools.github import fetch_easyblocks_from_commit, fetch_easyblocks_from_pr, fetch_github_token
 from easybuild.tools.hooks import KNOWN_HOOKS
-from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains
+from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains, \
+                                    include_job_backends
 from easybuild.tools.job.backend import avail_job_backends
 from easybuild.tools.modules import avail_modules_tools
 from easybuild.tools.module_generator import ModuleGeneratorLua, avail_module_generators
@@ -640,6 +641,8 @@ def config_options(self):
         opts = OrderedDict({
             'avail-module-naming-schemes': ("Show all supported module naming schemes",
                                             None, 'store_true', False,),
+            'avail-job-backends': ("Show all supported job backends",
+                                   None, "store_true", False,),
             'avail-modules-tools': ("Show all supported module tools",
                                     None, "store_true", False,),
             'avail-repositories': ("Show all repository types (incl. non-usable)",
@@ -668,6 +671,7 @@ def config_options(self):
                                               'strlist', 'store', []),
             'include-toolchains': ("Location(s) of extra or customized toolchains or toolchain components",
                                    'strlist', 'store', []),
+            'include-job-backends': ("Location(s) of extra or customized job backends", 'strlist', 'store', []),
             'installpath': ("Install path for software and modules",
                             None, 'store', mk_full_default_path('installpath')),
             'installpath-data': ("Install path for data (if None, combine --installpath and --subdir-data)",
@@ -1078,6 +1082,7 @@ def postprocess(self):
                 self.options.avail_modules_tools, self.options.avail_module_naming_schemes,
                 self.options.show_default_configfiles, self.options.avail_toolchain_opts,
                 self.options.avail_hooks, self.options.show_system_info,
+                self.options.avail_job_backends
                 )):
             build_easyconfig_constants_dict()  # runs the easyconfig constants sanity check
             self._postprocess_list_avail()
@@ -1208,6 +1213,9 @@ def _postprocess_include(self):
         if self.options.include_toolchains:
             include_toolchains(self.tmpdir, self.options.include_toolchains)
 
+        if self.options.include_job_backends:
+            include_job_backends(self.tmpdir, self.options.include_job_backends)
+
     def _postprocess_checks(self):
         """Check whether (combination of) configuration options make sense."""
 
@@ -1459,6 +1467,10 @@ def _postprocess_list_avail(self):
         if self.options.avail_module_naming_schemes:
             msg += self.avail_list('module naming schemes', avail_module_naming_schemes())
 
+        # dump supported job backends
+        if self.options.avail_job_backends:
+            msg += self.avail_list('job backends', avail_job_backends())
+
         # dump default list of config files that are considered
         if self.options.show_default_configfiles:
             msg += self.show_default_configfiles()

test/framework/include.py

@@ -34,7 +34,8 @@
 
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.filetools import mkdir, write_file
-from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains
+from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains, \
+                                    include_job_backends
 from easybuild.tools.include import is_software_specific_easyblock
 
 
@@ -243,6 +244,45 @@ def test_include_toolchains(self):
         my_tc_real_py_path = os.path.realpath(os.path.join(os.path.dirname(my_tc_pyc_path), 'my_tc.py'))
         self.assertTrue(os.path.samefile(up(my_tc_real_py_path, 1), my_toolchains))
 
+    def test_include_job_backend(self):
+        """Test include_job_backends()."""
+
+        my_job_backend = os.path.join(self.test_prefix, 'my_job_backend')
+        mkdir(my_job_backend)
+
+        # include __init__.py file that should be ignored, and shouldn't cause trouble (bug #1697)
+        write_file(os.path.join(my_job_backend, '__init__.py'), "# dummy init, should not get included")
+
+        my_job_backend_txt = '\n'.join([
+            "from easybuild.tools.job.backend import JobBackend",
+            "class MyJobBackend(JobBackend):",
+            "   pass",
+        ])
+        write_file(os.path.join(my_job_backend, 'my_job_backend.py'), my_job_backend_txt)
+
+        my_job_backend_bis = os.path.join(self.test_prefix, 'my_job_backend.py')
+        write_file(my_job_backend_bis, '')
+
+        # include custom job backend
+        included_job_backends_path = include_job_backends(self.test_prefix, [os.path.join(my_job_backend, '*.py'),
+                                                          my_job_backend_bis])
+
+        expected_paths = ['__init__.py', 'tools/__init__.py', 'tools/job/__init__.py',
+                          'tools/job/my_job_backend.py']
+        for filepath in expected_paths:
+            fullpath = os.path.join(included_job_backends_path, 'easybuild', filepath)
+            self.assertExists(fullpath)
+
+        # path to included job backends should be prepended to Python search path
+        self.assertEqual(sys.path[0], included_job_backends_path)
+
+        # importing custom job backends should work
+        import easybuild.tools.job.my_job_backend
+        my_job_backend_pyc_path = easybuild.tools.job.my_job_backend.__file__
+        my_job_backend_real_py_path = os.path.realpath(os.path.join(os.path.dirname(my_job_backend_pyc_path),
+                                                       'my_job_backend.py'))
+        self.assertTrue(os.path.samefile(up(my_job_backend_real_py_path, 1), my_job_backend))
+
     def test_is_software_specific_easyblock(self):
         """Test is_software_specific_easyblock function."""
 

test/framework/options.py

@@ -4183,6 +4183,58 @@ def test_include_toolchains(self):
             res = run_shell_cmd(test_cmd)
         self.assertRegex(res.output, tc_regex)
 
+    def test_include_job_backends(self):
+        """Test --include-job-backends."""
+
+        # make sure that calling out to 'eb' will work by restoring $PATH & $PYTHONPATH
+        self.restore_env_path_pythonpath()
+
+        topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+        # try and make sure 'eb' is available via $PATH if it isn't yet
+        path = self.env_path
+        if which('eb') is None:
+            path = '%s:%s' % (topdir, path)
+
+        # try and make sure top-level directory is in $PYTHONPATH if it isn't yet
+        pythonpath = self.env_pythonpath
+        with self.mocked_stdout_stderr():
+            res = run_shell_cmd("cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False)
+        if res.exit_code != 0:
+            pythonpath = '%s:%s' % (topdir, pythonpath)
+
+        fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
+        os.close(fd)
+
+        # clear log
+        write_file(self.logfile, '')
+
+        job_backend_regex = re.compile(r'^\s*TestIncludedJobBackend', re.M)
+
+        # TestIncludedJobBackend job backend is not available by default
+        args = ['--avail-job-backends']
+        test_cmd = self.mk_eb_test_cmd(args)
+        with self.mocked_stdout_stderr():
+            res = run_shell_cmd(test_cmd)
+        self.assertNotRegex(res.output, job_backend_regex)
+
+        # include extra test job backend
+        job_backend_txt = '\n'.join([
+            'from easybuild.tools.job.backend import JobBackend',
+            'class TestIncludedJobBackend(JobBackend):',
+            '   pass',
+        ])
+        write_file(os.path.join(self.test_prefix, 'test_job_backend.py'), job_backend_txt)
+
+        # clear log
+        write_file(self.logfile, '')
+
+        args.append('--include-job-backends=%s/*.py' % self.test_prefix)
+        test_cmd = self.mk_eb_test_cmd(args)
+        with self.mocked_stdout_stderr():
+            res = run_shell_cmd(test_cmd)
+        self.assertRegex(res.output, job_backend_regex)
+
     def test_cleanup_tmpdir(self):
         """Test --cleanup-tmpdir."""
         topdir = os.path.dirname(os.path.abspath(__file__))
@@ -5242,7 +5294,7 @@ def test_github_empty_pr(self):
             self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, do_build=True, raise_error=True)
 
     def test_show_config(self):
-        """"Test --show-config and --show-full-config."""
+        """Test --show-config and --show-full-config."""
 
         # only retain $EASYBUILD_* environment variables we expect for this test
         retained_eb_env_vars = [