Skip to content

Use assert(Not)Regex instead of assertTrue or assertFalse in tests #5188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Flamefire wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

Use assert(Not)Regex instead of assertTrue or assertFalse in tests #5188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3c13372
Fix non-interactive cmd.sh
Flamefire May 5, 2026
2cfc0dd
Add quotes to `echo` of `$BASH_ENV`
Flamefire May 5, 2026
3ffb7a9
Use `assert(Not)Regex` instead of `assertTrue` or `assertFalse` in tests
Flamefire May 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions easybuild/base/testing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
import difflib
import os
import pprint
import re
import sys
from contextlib import contextmanager
from io import StringIO
Expand Down Expand Up @@ -173,10 +172,7 @@ def assertErrorRegex(self, error, regex, call, *args, **kwargs):
str_args = ', '.join(list(map(str, args)) + str_kwargs)
self.fail("Expected errors with %s(%s) call should occur" % (call.__name__, str_args))
except error as err:
msg = self.convert_exception_to_str(err)
if isinstance(regex, str):
regex = re.compile(regex)
self.assertTrue(regex.search(msg), "Pattern '%s' is found in '%s'" % (regex.pattern, msg))
self.assertRegex(self.convert_exception_to_str(err), regex)

def mock_stdout(self, enable, force_tty=False):
"""Enable/disable mocking stdout."""
Expand Down
14 changes: 10 additions & 4 deletions easybuild/tools/run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,22 +273,28 @@ def create_cmd_scripts(cmd_str, work_dir, env, tmpdir, out_file, err_file):
# Make script that sets up bash shell with specified environment and working directory
cmd_fp = os.path.join(tmpdir, 'cmd.sh')

# using -i to force interactive shell, so env.sh is also sourced when -c is used to run commands
launch_cmd = 'bash --rcfile $EB_SCRIPT_DIR/env.sh -i "$@"'

launch_cmd = 'bash'
# prefix launch command with bwrap (if used)
bwrap_cmd = os.getenv('EB_BWRAP_CMD')
if bwrap_cmd:
launch_cmd = bwrap_cmd + ' ' + launch_cmd

# using -i for interactive shell, so env.sh is sourced
launch_cmd_interactive = f'{launch_cmd} --rcfile $EB_SCRIPT_DIR/env.sh -i'
launch_cmd_with_args = f'BASH_ENV=$EB_SCRIPT_DIR/env.sh {launch_cmd} "$@"'

with open(cmd_fp, 'w') as fid:
fid.write('#!/usr/bin/env bash\n')
fid.write('# Run this script to set up a shell environment that EasyBuild used to run the shell command\n')
fid.write('\n'.join([
'EB_SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )',
f'echo "# Shell for the command: \'"{shlex.quote(cmd_str)}"\'"',
'echo "# Use command history, exit to stop"',
launch_cmd,
'if [ "$#" -eq 0 ]; then',
' ' + launch_cmd_interactive,
'else',
' ' + launch_cmd_with_args,
'fi',
]))
os.chmod(cmd_fp, 0o775)

Expand Down
12 changes: 4 additions & 8 deletions test/framework/build_log.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,7 @@ def test_easybuilderror(self):
self.assertErrorRegex(EasyBuildError, 'BOOM', raise_easybuilderror, 'BOOM')
logtxt = read_file(logfile)

log_re = re.compile(r"^fancyroot ::.* BOOM \(at .*:[0-9]+ in [a-z_]+\)$", re.M)
self.assertTrue(log_re.match(logtxt), "%s matches %s" % (log_re.pattern, logtxt))
self.assertRegex(logtxt, re.compile(r"^fancyroot ::.* BOOM \(at .*:[0-9]+ in [a-z_]+\)$", re.M))

# test formatting of message
self.assertErrorRegex(EasyBuildError, 'BOOMBAF', raise_easybuilderror, 'BOOM%s', 'BAF')
Expand Down Expand Up @@ -154,8 +153,7 @@ def test_easybuildlog(self):
r"fancyroot.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops",
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
self.assertRegex(logtxt, re.compile(r'^%s' % expected_logtxt, re.M))

self.assertErrorRegex(EasyBuildError, r"DEPRECATED \(since .*: kaput", log.deprecated, "kaput", older_ver)
self.assertErrorRegex(EasyBuildError, r"DEPRECATED \(since .*: 2>1", log.deprecated, "2>1", '2.0', '1.0')
Expand All @@ -181,8 +179,7 @@ def test_easybuildlog(self):
r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild encountered an error \(at .* in .*\): foo baz baz",
'',
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
self.assertRegex(logtxt, re.compile(r'^%s' % expected_logtxt, re.M))

write_file(tmplog, '')
logToFile(tmplog, enable=True)
Expand Down Expand Up @@ -242,8 +239,7 @@ def test_log_levels(self):
error_msg, deprecated_msg, warning_msg, info_msg, debug_msg,
error_msg, deprecated_msg, warning_msg, info_msg, debug_msg, devel_msg,
])
logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M)
self.assertTrue(logtxt_regex.search(logtxt), "Pattern '%s' found in %s" % (logtxt_regex.pattern, logtxt))
self.assertRegex(logtxt, re.compile(r'^%s' % expected_logtxt, re.M))

def test_print_warning(self):
"""Test print_warning"""
Expand Down
23 changes: 8 additions & 15 deletions test/framework/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -598,33 +598,27 @@ def test_get_log_filename(self):
tmpdir = tempfile.gettempdir()

res = get_log_filename('foo', '1.2.3')
regex = re.compile(os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-[0-9]{8}\.[0-9]{6}\.log$'))
self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res))
self.assertRegex(res, re.compile(os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-[0-9]{8}\.[0-9]{6}\.log$')))

res = get_log_filename('foo', '1.2.3', date='19700101')
regex = re.compile(os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-19700101\.[0-9]{6}\.log$'))
self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res))
self.assertRegex(res, re.compile(os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-19700101\.[0-9]{6}\.log$')))

res = get_log_filename('foo', '1.2.3', timestamp='094651')
regex = re.compile(os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-[0-9]{8}\.094651\.log$'))
self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res))
self.assertRegex(res, re.compile(os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-[0-9]{8}\.094651\.log$')))

res = get_log_filename('foo', '1.2.3', date='19700101', timestamp='094651')
regex = re.compile(os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-19700101\.094651\.log$'))
self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res))
self.assertRegex(res, re.compile(os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-19700101\.094651\.log$')))

# if log file already exists, numbers are added to the filename to obtain a new file path
write_file(res, '')
res = get_log_filename('foo', '1.2.3', date='19700101', timestamp='094651')
regex = re.compile(os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-19700101\.094651\.log\.1$'))
self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res))
self.assertRegex(res, re.compile(os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-19700101\.094651\.log\.1$')))

# adding salt ensures a unique filename (pretty much)
prev_log_filenames = []
for i in range(10):
for _ in range(10):
res = get_log_filename('foo', '1.2.3', date='19700101', timestamp='094651', add_salt=True)
regex = re.compile(os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-19700101\.094651\.[a-zA-Z]{5}\.log$'))
self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res))
self.assertRegex(res, os.path.join(tmpdir, r'easybuild-foo-1\.2\.3-19700101\.094651\.[a-zA-Z]{5}\.log$'))
self.assertNotIn(res, prev_log_filenames)
prev_log_filenames.append(res)

Expand Down Expand Up @@ -676,9 +670,8 @@ def test_log_path(self):

# reconfigure with value for log directory that includes templates
init_config(args=['--logfile-format=easybuild-%(name)s-%(version)s-%(date)s-%(time)s,log.txt'])
regex = re.compile(r'^easybuild-foo-1\.2\.3-[0-9-]{8}-[0-9]{6}$')
res = log_path(ec=ec)
self.assertTrue(regex.match(res), "Pattern '%s' matches '%s'" % (regex.pattern, res))
self.assertRegex(res, r'^easybuild-foo-1\.2\.3-[0-9-]{8}-[0-9]{6}$')
self.assertEqual(log_file_format(), 'log.txt')

def test_get_build_log_path(self):
Expand Down
64 changes: 23 additions & 41 deletions test/framework/containers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,12 +84,6 @@ def run_main(self, args, raise_error=True):

return stdout, stderr

def check_regexs(self, regexs, stdout):
"""Helper function to check output of stdout."""
for regex in regexs:
regex = re.compile(regex, re.M)
self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))

def test_end2end_singularity_recipe_config(self):
"""End-to-end test for --containerize (recipe only), using --container-config."""
test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
Expand Down Expand Up @@ -122,7 +116,7 @@ def test_end2end_singularity_recipe_config(self):
self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True)

args[-1] = 'bootstrap=yum,osversion=7.6.1810'
stdout, stderr = self.run_main(args, raise_error=True)
self.run_main(args, raise_error=True)

txt = read_file(test_container_recipe)
expected = '\n'.join([
Expand All @@ -137,8 +131,7 @@ def test_end2end_singularity_recipe_config(self):
# when installing from scratch, a bunch of OS packages are installed too
pkgs = ['epel-release', 'python', 'setuptools', 'Lmod', r'gcc-c\+\+', 'make', 'patch', 'tar']
for pkg in pkgs:
regex = re.compile(r"^yum install .*%s" % pkg, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
self.assertRegex(txt, re.compile(r"^yum install .*%s" % pkg, re.M))

pip_patterns = [
# EasyBuild is installed with pip3 by default
Expand All @@ -152,15 +145,13 @@ def test_end2end_singularity_recipe_config(self):
r"if \[ ! -d /scratch \]; then mkdir -p /scratch",
]
eb_pattern = r"eb toy-0.0.eb --robot\s*$"
for pattern in pip_patterns + post_commands_patterns + [eb_pattern]:
regex = re.compile('^' + pattern, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
self.assert_multi_regex(pip_patterns + post_commands_patterns + [eb_pattern], txt)

remove_file(test_container_recipe)

# can also specify a custom mirror URL
args[-1] += ',mirrorurl=https://example.com'
stdout, stderr = self.run_main(args, raise_error=True)
self.run_main(args, raise_error=True)

txt = read_file(test_container_recipe)
expected = '\n'.join([
Expand Down Expand Up @@ -213,10 +204,9 @@ def test_end2end_singularity_recipe_config(self):

# no OS packages are installed by default when starting from an existing image
self.assertNotIn("yum install", txt)

for pattern in pip_patterns + post_commands_patterns + [eb_pattern]:
regex = re.compile('^' + pattern, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
self.assert_multi_regex(
(f'^{pattern}' for pattern in pip_patterns + post_commands_patterns + [eb_pattern]),
txt)

remove_file(test_container_recipe)

Expand All @@ -226,12 +216,10 @@ def test_end2end_singularity_recipe_config(self):
txt = read_file(test_container_recipe)

for pattern in pip_patterns:
regex = re.compile('^' + pattern, re.M)
self.assertFalse(regex.search(txt), "Pattern '%s' should not be found in: %s" % (regex.pattern, txt))
self.assertNotRegex(txt, re.compile('^' + pattern, re.M))

for pattern in ["easy_install easybuild", eb_pattern]:
regex = re.compile('^' + pattern, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt))
self.assertRegex(txt, re.compile('^' + pattern, re.M))

remove_file(test_container_recipe)

Expand All @@ -241,12 +229,10 @@ def test_end2end_singularity_recipe_config(self):
txt = read_file(test_container_recipe)

for pattern in post_commands_patterns:
regex = re.compile('^' + pattern, re.M)
self.assertFalse(regex.search(txt), "Pattern '%s' should not be found in: %s" % (regex.pattern, txt))
self.assertNotRegex(txt, re.compile('^' + pattern, re.M))

for pattern in ["id easybuild", eb_pattern]:
regex = re.compile('^' + pattern, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt))
self.assertRegex(txt, re.compile('^' + pattern, re.M))

remove_file(test_container_recipe)

Expand All @@ -255,8 +241,7 @@ def test_end2end_singularity_recipe_config(self):
stdout, stderr = self.run_main(args, raise_error=True)
txt = read_file(test_container_recipe)

regex = re.compile(r"^eb toy-0.0.eb --robot --debug -l", re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt))
self.assertRegex(txt, re.compile(r"^eb toy-0.0.eb --robot --debug -l", re.M))

def test_end2end_singularity_image(self):
"""End-to-end test for --containerize (recipe + image)."""
Expand Down Expand Up @@ -306,7 +291,7 @@ def test_end2end_singularity_image(self):
r"^== Running 'sudo\s*\S*/singularity build\s*/.* /.*', you may need to enter your 'sudo' password...",
r"^== Singularity image created at %s/containers/toy-0.0\.%s" % (self.test_prefix, ext),
]
self.check_regexs(regexs, stdout)
self.assert_multi_regex(regexs, stdout)

self.assertExists(os.path.join(containerpath, 'toy-0.0.%s' % ext))

Expand All @@ -327,7 +312,7 @@ def test_end2end_singularity_image(self):
r"^== Running 'sudo\s*\S*/singularity build --writable /.* /.*', you may need to enter .*",
r"^== Singularity image created at %s/containers/foo-bar\.img$" % self.test_prefix,
]
self.check_regexs(regexs, stdout)
self.assert_multi_regex(regexs, stdout)

cont_img = os.path.join(containerpath, 'foo-bar.img')
self.assertExists(cont_img)
Expand All @@ -347,21 +332,21 @@ def test_end2end_singularity_image(self):
regexs.extend([
"WARNING: overwriting existing container image at %s due to --force" % cont_img,
])
self.check_regexs(regexs, stdout)
self.assert_multi_regex(regexs, stdout)
self.assertExists(cont_img)

# also check behaviour under --extended-dry-run
args.append('--extended-dry-run')
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
self.check_regexs(regexs, stdout)
self.assert_multi_regex(regexs, stdout)

# test use of --container-tmpdir
args.append('--container-tmpdir=%s' % self.test_prefix)
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
regexs[-3] = r"^== Running 'sudo\s*SINGULARITY_TMPDIR=%s \S*/singularity build .*" % self.test_prefix
self.check_regexs(regexs, stdout)
self.assert_multi_regex(regexs, stdout)

def test_end2end_dockerfile(self):
test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
Expand Down Expand Up @@ -390,7 +375,7 @@ def test_end2end_dockerfile(self):
stdout, stderr = self.run_main(base_args + ['--container-config=%s' % cont_base])
self.assertFalse(stderr)
regexs = ["^== Dockerfile definition file created at %s/containers/Dockerfile.toy-0.0" % self.test_prefix]
self.check_regexs(regexs, stdout)
self.assert_multi_regex(regexs, stdout)
remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0'))

self.run_main(base_args + ['--container-config=centos:7'])
Expand All @@ -413,12 +398,10 @@ def test_end2end_dockerfile(self):
"eb --robot toy-0.0.eb GCC-4.9.2.eb",
"module load toy/0.0 GCC/4.9.2",
]
self.check_regexs(regexs, def_file)
self.assert_multi_regex(regexs, def_file)

# there should be no leading/trailing whitespace included
for pattern in [r'^\s+', r'\s+$']:
regex = re.compile(pattern)
self.assertFalse(regex.search(def_file), "Pattern '%s' should *not* be found in: %s" % (pattern, def_file))
self.assert_multi_regex((r'^\s+', r'\s+$'), def_file, assert_true=False, flags=0)

def test_end2end_docker_image(self):

Expand Down Expand Up @@ -462,12 +445,12 @@ def test_end2end_docker_image(self):
r"^== Running 'sudo docker build -f .* -t .* \.', you may need to enter your 'sudo' password...",
r"^== Docker image created at toy-0.0:latest",
]
self.check_regexs(regexs, stdout)
self.assert_multi_regex(regexs, stdout)

args.extend(['--force', '--extended-dry-run'])
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
self.check_regexs(regexs, stdout)
self.assert_multi_regex(regexs, stdout)

def test_container_config_template_recipe(self):
"""Test use of --container-config and --container-template-recipe."""
Expand All @@ -494,8 +477,7 @@ def test_container_config_template_recipe(self):
stdout, stderr = self.run_main(args)

self.assertFalse(stderr)
regex = re.compile("^== Singularity definition file created at .*/containers/Singularity.toy-0.0$")
self.assertTrue(regex.match(stdout), "Stdout matches pattern '%s': %s" % (regex.pattern, stdout))
self.assertRegex(stdout, "^== Singularity definition file created at .*/containers/Singularity.toy-0.0$")

expected = '\n'.join([
"# this is just a test",
Expand Down
Loading
Loading