From 62195e3619312ec03437034cfe541b03d096f68a Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Fri, 14 Jan 2022 01:33:14 +0100 Subject: [PATCH 01/32] feat(typing): add typing extensions and stubs --- .github/workflows/reviewdog.yml | 5 +++++ mypy.ini | 1 + test-requirements.txt | 2 ++ 3 files changed, 8 insertions(+) diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index 61f7e0627..d94b8fed2 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -18,6 +18,11 @@ jobs: pip install . pip install mypy pip install pylint + pip install sqlalchemy[mypy] + pip install types-Jinja2 + pip install types-PyYAML + pip install types-six + pip install types-setuptools - name: Setup reviewdog run: | diff --git a/mypy.ini b/mypy.ini index 79ed17c78..9a577dd0f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -10,6 +10,7 @@ warn_return_any = True warn_unreachable = True warn_unused_configs = True warn_unused_ignores = True +plugins = sqlalchemy.ext.mypy.plugin [mypy-plumbum.*] ignore_missing_imports = True diff --git a/test-requirements.txt b/test-requirements.txt index 575414af0..7f955153a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,3 +5,5 @@ mock pytest pytest-cov pytest-describe +sqlalchemy[mypy] +yapf From 0a125f445b4d9499cdaa08d1bc668faa69a79098 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Fri, 14 Jan 2022 01:33:39 +0100 Subject: [PATCH 02/32] feat(typing): add/fix mypy type annotations --- benchbuild/cli/slurm.py | 2 +- benchbuild/environments/entrypoints/cli.py | 31 ++++---- benchbuild/extensions/base.py | 8 +- benchbuild/projects/gentoo/__init__.py | 28 +++++-- benchbuild/projects/gentoo/gentoo.py | 6 +- benchbuild/projects/lnt/lnt.py | 74 +++++++++++-------- .../projects/polybench/polybench-mod.py | 20 +++-- benchbuild/utils/actions.py | 40 ++++++---- benchbuild/utils/bootstrap.py | 16 ++-- benchbuild/utils/dict.py | 11 ++- benchbuild/utils/run.py | 6 +- benchbuild/utils/schedule_tree.py | 31 ++++---- benchbuild/utils/settings.py | 12 +-- 13 files changed, 172 insertions(+), 113 deletions(-) diff --git a/benchbuild/cli/slurm.py b/benchbuild/cli/slurm.py index 34bead75b..0ba5c8545 100644 --- a/benchbuild/cli/slurm.py +++ b/benchbuild/cli/slurm.py @@ -51,7 +51,7 @@ def experiment_tag(self, description): list=True, requires=["--experiment"], help="Run a group of projects under the given experiments") - def group(self, groups: tp.List[str]) -> None: # type: ignore + def group(self, groups: tp.List[str]) -> None: """Run a group of projects under the given experiments""" self.group_args = groups diff --git a/benchbuild/environments/entrypoints/cli.py b/benchbuild/environments/entrypoints/cli.py index 41b1d401f..54c432a42 100644 --- a/benchbuild/environments/entrypoints/cli.py +++ b/benchbuild/environments/entrypoints/cli.py @@ -15,7 +15,7 @@ rich.traceback.install() -class BenchBuildContainer(cli.Application): # type: ignore +class BenchBuildContainer(cli.Application): """ Top-level command for benchbuild containers. """ @@ -30,24 +30,23 @@ def main(self, *args: str) -> int: @BenchBuildContainer.subcommand("run") -class BenchBuildContainerRun(cli.Application): # type: ignore +class BenchBuildContainerRun(cli.Application): experiment_args: tp.List[str] = [] group_args: tp.List[str] = [] @cli.switch(["-E", "--experiment"], str, list=True, - help="Specify experiments to run") # type: ignore - def set_experiments(self, names: tp.List[str]) -> None: # type: ignore + help="Specify experiments to run") + def set_experiments(self, names: tp.List[str]) -> None: self.experiment_args = names @cli.switch(["-G", "--group"], str, list=True, requires=["--experiment"], - help="Run a group of projects under the given experiments" - ) # type: ignore - def set_group(self, groups: tp.List[str]) -> None: # type: ignore + help="Run a group of projects under the given experiments") + def set_group(self, groups: tp.List[str]) -> None: self.group_args = groups image_export = cli.Flag(['export'], @@ -143,17 +142,16 @@ class BenchBuildContainerBase(cli.Application): @cli.switch(["-E", "--experiment"], str, list=True, - help="Specify experiments to run") # type: ignore - def set_experiments(self, names: tp.List[str]) -> None: # type: ignore + help="Specify experiments to run") + def set_experiments(self, names: tp.List[str]) -> None: self.experiment_args = names @cli.switch(["-G", "--group"], str, list=True, requires=["--experiment"], - help="Run a group of projects under the given experiments" - ) # type: ignore - def set_group(self, groups: tp.List[str]) -> None: # type: ignore + help="Run a group of projects under the given experiments") + def set_group(self, groups: tp.List[str]) -> None: self.group_args = groups image_export = cli.Flag(['export'], @@ -222,17 +220,16 @@ class BenchBuildContainerRemoveImages(cli.Application): @cli.switch(["-E", "--experiment"], str, list=True, - help="Specify experiments to run") # type: ignore - def set_experiments(self, names: tp.List[str]) -> None: # type: ignore + help="Specify experiments to run") + def set_experiments(self, names: tp.List[str]) -> None: self.experiment_args = names @cli.switch(["-G", "--group"], str, list=True, requires=["--experiment"], - help="Run a group of projects under the given experiments" - ) # type: ignore - def set_group(self, groups: tp.List[str]) -> None: # type: ignore + help="Run a group of projects under the given experiments") + def set_group(self, groups: tp.List[str]) -> None: self.group_args = groups delete_project_images = cli.Flag(['with-projects'], diff --git a/benchbuild/extensions/base.py b/benchbuild/extensions/base.py index 449ebde0e..48bb5c6f5 100644 --- a/benchbuild/extensions/base.py +++ b/benchbuild/extensions/base.py @@ -1,6 +1,8 @@ """ Extension base-classes for compile-time and run-time experiments. """ +from __future__ import annotations + import logging import typing as tp from abc import ABCMeta @@ -35,7 +37,7 @@ class Extension(metaclass=ABCMeta): def __init__( self, - *extensions: 'Extension', + *extensions: Extension, config: tp.Optional[tp.Dict[str, str]] = None, **kwargs: tp.Any ): @@ -73,8 +75,8 @@ def call_next(self, *args: tp.Any, return all_results - def __lshift__(self, rhs: 'Extension') -> 'Extension': - rhs.next_extensions = [self] + def __lshift__(self, rhs: Extension) -> Extension: + rhs.next_extensions = tuple([self]) return rhs def print(self, indent: int = 0) -> None: diff --git a/benchbuild/projects/gentoo/__init__.py b/benchbuild/projects/gentoo/__init__.py index cca5463b4..e510a1211 100644 --- a/benchbuild/projects/gentoo/__init__.py +++ b/benchbuild/projects/gentoo/__init__.py @@ -10,8 +10,20 @@ from benchbuild.settings import CFG -from . import (autoportage, bzip2, crafty, eix, gentoo, gzip, info, lammps, - postgresql, sevenz, x264, xz) +from . import ( + autoportage, + bzip2, + crafty, + eix, + gentoo, + gzip, + info, + lammps, + postgresql, + sevenz, + x264, + xz, +) LOG = logging.getLogger(__name__) @@ -24,11 +36,13 @@ def __initialize_dynamic_projects__(autotest_path): with open(autotest_path, 'r') as ebuilds: for line in ebuilds: ebuild_data = line.strip('\n') - ebuild_data = ebuild_data.split('/') - domain = ebuild_data[0] - name = ebuild_data[1] - PortageFactory("Auto{0}{1}".format(domain, name), - domain + "_" + name, domain) + ebuild_data_list = ebuild_data.split('/') + domain = ebuild_data_list[0] + name = ebuild_data_list[1] + PortageFactory( + "Auto{0}{1}".format(domain, name), domain + "_" + name, + domain + ) __initialize_dynamic_projects__(str(CFG['gentoo']['autotest_loc'])) diff --git a/benchbuild/projects/gentoo/gentoo.py b/benchbuild/projects/gentoo/gentoo.py index 68683fa85..3cc3322a9 100644 --- a/benchbuild/projects/gentoo/gentoo.py +++ b/benchbuild/projects/gentoo/gentoo.py @@ -36,7 +36,7 @@ class GentooGroup(bb.Project): SRC_FILE = None CONTAINER = declarative.ContainerImage().from_('benchbuild:alpine') - emerge_env: tp.Dict[str, tp.Any] = attr.ib( + emerge_env: tp.MutableMapping[str, tp.Any] = attr.ib( default=attr.Factory(dict), repr=False, eq=False, order=False ) @@ -62,7 +62,7 @@ def compile(self) -> None: ) LOG.debug('Installing dependencies.') - emerge(package_atom, '--onlydeps', env=self.emerge_env) + emerge(package_atom, '--onlydeps', **self.emerge_env) c_compiler = local.path(str(compiler.cc(self))) cxx_compiler = local.path(str(compiler.cxx(self))) @@ -71,7 +71,7 @@ def compile(self) -> None: ln('-sf', str(cxx_compiler), local.path('/') / cxx_compiler.basename) LOG.debug('Installing %s.', package_atom) - emerge(package_atom, env=self.emerge_env) + emerge(package_atom, **self.emerge_env) def configure_benchbuild(self, cfg: Configuration) -> None: config_file = local.path("/.benchbuild.yml") diff --git a/benchbuild/projects/lnt/lnt.py b/benchbuild/projects/lnt/lnt.py index c22453228..d9dc6d29f 100644 --- a/benchbuild/projects/lnt/lnt.py +++ b/benchbuild/projects/lnt/lnt.py @@ -20,16 +20,20 @@ class LNTGroup(bb.Project): r'(?P.+)\.simple', r'(?P.+)-(dbl|flt)', ] - SUBDIR = None + SUBDIR = '' SOURCE = [ - Git(remote='http://llvm.org/git/lnt', + Git( + remote='http://llvm.org/git/lnt', local='lnt.git', refspec='HEAD', - limit=5), - Git(remote='http://llvm.org/git/test-suite', + limit=5 + ), + Git( + remote='http://llvm.org/git/test-suite', local='test-suite', refspec='HEAD', - limit=5) + limit=5 + ) ] # Will be set by configure. @@ -49,8 +53,10 @@ def compile(self): pip = local[pip_path] with local.cwd(lnt_repo): - pip("install", "--no-cache-dir", "--disable-pip-version-check", - "-e", ".") + pip( + "install", "--no-cache-dir", "--disable-pip-version-check", + "-e", "." + ) self.sandbox_dir = local.cwd / "run" if self.sandbox_dir.exists(): @@ -62,11 +68,12 @@ def compile(self): self.clang_cxx = bb.compiler.cxx(self, detect_project=True) _runtest = bb.watch(self.lnt) - _runtest("runtest", "test-suite", "-v", "-j1", "--sandbox", - self.sandbox_dir, "--benchmarking-only", - "--only-compile", "--cc", str(self.clang), "--cxx", - str(self.clang_cxx), "--test-suite", test_suite_source, - "--only-test=" + self.SUBDIR) + _runtest( + "runtest", "test-suite", "-v", "-j1", "--sandbox", + self.sandbox_dir, "--benchmarking-only", "--only-compile", "--cc", + str(self.clang), "--cxx", str(self.clang_cxx), "--test-suite", + test_suite_source, "--only-test=" + self.SUBDIR + ) @staticmethod def after_run_tests(sandbox_dir): @@ -77,17 +84,18 @@ def after_run_tests(sandbox_dir): def run_tests(self): test_suite_source = local.path(self.source_of('test-suite')) - binary = bb.wrapping.wrap_dynamic(self, - "lnt_runner", - name_filters=LNTGroup.NAME_FILTERS) + binary = bb.wrapping.wrap_dynamic( + self, "lnt_runner", name_filters=LNTGroup.NAME_FILTERS + ) _runtest = bb.watch(self.lnt) - _runtest("runtest", "nt", "-v", "-j1", "--sandbox", self.sandbox_dir, - "--benchmarking-only", "--cc", str(self.clang), "--cxx", - str(self.clang_cxx), "--test-suite", test_suite_source, - "--test-style", "simple", "--test-externals", self.builddir, - "--make-param=RUNUNDER=" + str(binary), - "--only-test=" + self.SUBDIR) + _runtest( + "runtest", "nt", "-v", "-j1", "--sandbox", self.sandbox_dir, + "--benchmarking-only", "--cc", str(self.clang), "--cxx", + str(self.clang_cxx), "--test-suite", test_suite_source, + "--test-style", "simple", "--test-externals", self.builddir, + "--make-param=RUNUNDER=" + str(binary), "--only-test=" + self.SUBDIR + ) LNTGroup.after_run_tests(self.sandbox_dir) @@ -120,8 +128,10 @@ def compile(self): super(SPEC2006, self).compile() else: print('======================================================') - print(('SPECCPU2006 not found in %s. This project will fail.', - CFG['tmp_dir'])) + print(( + 'SPECCPU2006 not found in %s. This project will fail.', + CFG['tmp_dir'] + )) print('======================================================') @@ -130,16 +140,22 @@ class Povray(LNTGroup): DOMAIN = 'LNT (Ext)' SUBDIR = "External/Povray" SOURCE = [ - Git(remote='http://llvm.org/git/lnt', + Git( + remote='http://llvm.org/git/lnt', local='lnt.git', refspec='HEAD', - limit=5), - Git(remote='http://llvm.org/git/test-suite', + limit=5 + ), + Git( + remote='http://llvm.org/git/test-suite', local='test-suite', refspec='HEAD', - limit=5), - Git(remote='https://github.com/POV-Ray/povray', + limit=5 + ), + Git( + remote='https://github.com/POV-Ray/povray', local='povray.git', refspec='HEAD', - limit=5) + limit=5 + ) ] diff --git a/benchbuild/projects/polybench/polybench-mod.py b/benchbuild/projects/polybench/polybench-mod.py index 82975ac8e..329e24916 100644 --- a/benchbuild/projects/polybench/polybench-mod.py +++ b/benchbuild/projects/polybench/polybench-mod.py @@ -11,10 +11,12 @@ class PolybenchModGroup(PolyBenchGroup): GROUP = 'polybench-mod' DOMAIN = 'polybench' SOURCE = [ - Git(remote='https://github.com/simbuerg/polybench-c-4.2-1.git', + Git( + remote='https://github.com/simbuerg/polybench-c-4.2-1.git', local='polybench.git', limit=5, - refspec='HEAD') + refspec='HEAD' + ) ] def compile(self): @@ -31,22 +33,24 @@ def compile(self): kernel_file = src_sub / (self.name + "_kernel.c") utils_dir = src_dir / "utilities" - polybench_opts = [ + polybench_compile_options = [ "-D" + str(workload), "-DPOLYBENCH_USE_C99_PROTO", "-DPOLYBENCH_USE_RESTRICT" ] if verify: - polybench_opts = self.compile_verify([ + polybench_compile_options = self.compile_verify([ "-I", utils_dir, "-I", src_sub, utils_dir / "polybench.c", kernel_file, src_file, "-lm" - ], polybench_opts) + ], polybench_compile_options) clang = bb.compiler.cc(self) _clang = bb.watch(clang) - _clang("-I", utils_dir, "-I", src_sub, polybench_opts, - utils_dir / "polybench.c", kernel_file, src_file, "-lm", "-o", - self.name) + _clang( + "-I", utils_dir, "-I", src_sub, polybench_compile_options, + utils_dir / "polybench.c", kernel_file, src_file, "-lm", "-o", + self.name + ) class Correlation(PolybenchModGroup): diff --git a/benchbuild/utils/actions.py b/benchbuild/utils/actions.py index 5dfff34d1..fb8765b98 100644 --- a/benchbuild/utils/actions.py +++ b/benchbuild/utils/actions.py @@ -12,6 +12,8 @@ ```python ``` """ +from __future__ import annotations + import enum import functools as ft import itertools @@ -98,11 +100,11 @@ def prepend_status(func: DecoratedFunction[str]) -> DecoratedFunction[str]: """Prepends the output of `func` with the status.""" @tp.overload - def wrapper(self: 'Step', indent: int) -> str: + def wrapper(self: Step, indent: int) -> str: ... @tp.overload - def wrapper(self: 'Step') -> str: + def wrapper(self: Step) -> str: ... @ft.wraps(func) @@ -123,7 +125,7 @@ def notify_step_begin_end( @ft.wraps(func) def wrapper( - self: 'Step', *args: tp.Any, **kwargs: tp.Any + self: Step, *args: tp.Any, **kwargs: tp.Any ) -> StepResultVariants: """Wrapper stub.""" cls = self.__class__ @@ -172,7 +174,7 @@ class StepClass(type): """Decorate `steps` with logging and result conversion.""" def __new__( - mcs: tp.Type['StepClass'], name: str, bases: tp.Tuple[type, ...], + mcs: tp.Type[StepClass], name: str, bases: tp.Tuple[type, ...], attrs: tp.Dict[str, tp.Any] ) -> tp.Any: if not 'NAME' in attrs: @@ -231,8 +233,9 @@ class Step(metaclass=StepClass): NAME: tp.ClassVar[str] = "" DESCRIPTION: tp.ClassVar[str] = "" - ON_STEP_BEGIN = [] - ON_STEP_END = [] + ON_STEP_BEGIN: tp.List[tp.Callable[[Step], None]] = [] + ON_STEP_END: tp.List[tp.Callable[[Step, tp.Callable[[Step], None]], + None]] = [] obj = attr.ib(default=None, repr=False) action_fn = attr.ib(default=None, repr=False) @@ -421,7 +424,7 @@ class Any(Step): actions = attr.ib(default=attr.Factory(list), repr=False, eq=False) - def __len__(self) -> int: + def __len__(self): return sum([len(x) for x in self.actions]) + 1 def __iter__(self): @@ -446,8 +449,10 @@ def __call__(self) -> tp.List[StepResult]: def __str__(self, indent: int = 0) -> str: sub_actns = [a.__str__(indent + 1) for a in self.actions] - sub_actns = "\n".join(sub_actns) - return textwrap.indent("* Execute all of:\n" + sub_actns, indent * " ") + sub_actns_str = "\n".join(sub_actns) + return textwrap.indent( + "* Execute all of:\n" + sub_actns_str, indent * " " + ) @attr.s(eq=False, hash=True) @@ -510,7 +515,8 @@ def __run_children(self, num_processes: int) -> tp.List[StepResult]: LOG.info("Experiment aborting by user request") results.append(StepResult.ERROR) except Exception: - LOG.error("Experiment terminates " "because we got an exception:") + LOG.error("Experiment terminates " + "because we got an exception:") e_type, e_value, e_traceb = sys.exc_info() lines = traceback.format_exception(e_type, e_value, e_traceb) LOG.error("".join(lines)) @@ -532,9 +538,9 @@ def __call__(self) -> tp.List[StepResult]: def __str__(self, indent: int = 0) -> str: sub_actns = [a.__str__(indent + 1) for a in self.actions] - sub_actns = "\n".join(sub_actns) + sub_actns_str = "\n".join(sub_actns) return textwrap.indent( - "\nExperiment: {0}\n".format(self.obj.name) + sub_actns, + "\nExperiment: {0}\n".format(self.obj.name) + sub_actns_str, indent * " " ) @@ -546,7 +552,7 @@ class RequireAll(Step): actions = attr.ib(default=attr.Factory(list)) - def __len__(self): + def __len__(self) -> int: return sum([len(x) for x in self.actions]) + 1 def __iter__(self): @@ -593,8 +599,10 @@ def __call__(self) -> StepResultVariants: def __str__(self, indent: int = 0) -> str: sub_actns = [a.__str__(indent + 1) for a in self.actions] - sub_actns = "\n".join(sub_actns) - return textwrap.indent("* All required:\n" + sub_actns, indent * " ") + sub_actns_str = "\n".join(sub_actns) + return textwrap.indent( + "* All required:\n" + sub_actns_str, indent * " " + ) @attr.s @@ -681,7 +689,7 @@ def __call__(self) -> None: def __str__(self, indent: int = 0) -> str: project = self.obj variant = project.variant - version_str = source.to_str(tuple(variant.values())) + version_str = source.to_str(*tuple(variant.values())) return textwrap.indent( "* Project environment for: {} @ {}".format( diff --git a/benchbuild/utils/bootstrap.py b/benchbuild/utils/bootstrap.py index bd6727ff7..5a06aa92b 100644 --- a/benchbuild/utils/bootstrap.py +++ b/benchbuild/utils/bootstrap.py @@ -184,24 +184,30 @@ def install_package(pkg_name: str) -> bool: return False if pkg_name not in PACKAGES: - print("No bootstrap support for package '{0}'".format(pkg_name)) + print(f'No bootstrap support for package "{pkg_name}"') linux = linux_distribution_major() + if linux is None: + print(f'No bootstrap support for {linux}') + return False + package_manager = PACKAGE_MANAGER[linux] packages = PACKAGES[pkg_name][linux] + + ret = True for pkg_name_on_host in packages: - print("You are missing the package: '{0}'".format(pkg_name_on_host)) + print(f'You are missing the package: "{pkg_name_on_host}"') cmd = local["sudo"] cmd = cmd[package_manager["cmd"], package_manager["args"], pkg_name_on_host] cmd_str = str(cmd) - ret = False - if ui.ask("Run '{cmd}' to install it?".format(cmd=cmd_str)): - print("Running: '{cmd}'".format(cmd=cmd_str)) + if ui.ask(f'Run "{cmd_str}" to install it?'): + print(f'Running: "{cmd_str}"') try: (cmd & FG(retcode=0)) except ProcessExecutionError: + ret = False print("NOT INSTALLED") else: print("OK") diff --git a/benchbuild/utils/dict.py b/benchbuild/utils/dict.py index 344158aa6..2baf5b59c 100644 --- a/benchbuild/utils/dict.py +++ b/benchbuild/utils/dict.py @@ -1,8 +1,13 @@ """An extensible dictionary.""" +import typing as tp from contextlib import contextmanager -def extend_as_list(original_dict, **kwargs): +def extend_as_list(original_dict: tp.MutableMapping[tp.Any, tp.Any], + **kwargs) -> tp.Dict[tp.Any, tp.Any]: + """ + Extend values in a map by treating them as a list. + """ new_dict = original_dict for k, v in kwargs.items(): if k in original_dict: @@ -19,13 +24,13 @@ def extend_as_list(original_dict, **kwargs): new_dict[k] = oldv else: new_dict[k] = v - return new_dict + return dict(new_dict) class ExtensibleDict: """A dictionary that provides temporary modification.""" - _current = dict() + _current: tp.MutableMapping[tp.Any, tp.Any] = {} _extender_fn = None def __init__(self, extender_fn=None): diff --git a/benchbuild/utils/run.py b/benchbuild/utils/run.py index 6bb96c56f..a14f8f570 100644 --- a/benchbuild/utils/run.py +++ b/benchbuild/utils/run.py @@ -267,7 +267,9 @@ def fail_run_group(group, session): session.commit() -def exit_code_from_run_infos(run_infos: t.List[RunInfo]) -> int: +def exit_code_from_run_infos( + run_infos: t.Union[RunInfo, t.List[RunInfo]] +) -> int: """Generate a single exit code from a list of RunInfo objects. Takes a list of RunInfos and returns the exit code that is furthest away @@ -281,7 +283,7 @@ def exit_code_from_run_infos(run_infos: t.List[RunInfo]) -> int: """ assert run_infos is not None - if not hasattr(run_infos, "__iter__"): + if isinstance(run_infos, RunInfo): return run_infos.retcode rcs = [ri.retcode for ri in run_infos] diff --git a/benchbuild/utils/schedule_tree.py b/benchbuild/utils/schedule_tree.py index 0e0598368..cae92d483 100644 --- a/benchbuild/utils/schedule_tree.py +++ b/benchbuild/utils/schedule_tree.py @@ -1,6 +1,7 @@ """ Parsing utilities for Polly's ScheduleTree representation. """ import logging import textwrap as t +import typing as tp import attr import pyparsing as p @@ -12,32 +13,32 @@ class Node: tok = attr.ib() - def indent(self, level=0, idt=' '): + def indent(self, level: int = 0, idt: str = ' ') -> str: val = self.tok[2] if not isinstance(self.tok[2], str): val = self.tok[2].indent(1) - return t.indent('"{:s}": "{:s}"'.format(self.tok[0], val), level * idt) + return t.indent(f'"{self.tok[0]}": "{val}"', level * idt) @attr.s class CoincidenceNode(Node): - def indent(self, level=0, idt=' '): + def indent(self, level: int = 0, idt: str = ' ') -> str: ret = [str(child) for child in self.tok[3]] - ret = ",".join(ret) + ret_str = ",".join(ret) - return t.indent('"{:s}": [{:s}]'.format(self.tok[0], ret), level * idt) + return t.indent(f'"{self.tok[0]}": [{ret_str}]', level * idt) @attr.s class RootNode(Node): - def indent(self, level=0, idt=' '): + def indent(self, level: int = 0, idt: str = ' ') -> str: ret = [] ret = [child.indent(level + 2) for child in self.tok[1]] - ret = ",\n".join(ret) + ret_str = ",\n".join(ret) - return t.indent('{{\n{:s}\n}}'.format(ret), level * idt) + return t.indent(f'{{\n{ret_str}\n}}', level * idt) def __str__(self): return self.indent(0) @@ -46,7 +47,7 @@ def __str__(self): @attr.s class ChildNode(Node): - def indent(self, level=0, idt=' '): + def indent(self, level: int = 0, idt: str = ' ') -> str: ret = self.tok[0].indent(level) return ret @@ -54,8 +55,8 @@ def indent(self, level=0, idt=' '): @attr.s class SequenceNode(Node): - def indent(self, level=0, idt=' '): - ret = '"{:s}": [\n'.format(self.tok[0]) + def indent(self, level: int = 0, idt: str = ' ') -> str: + ret = f'"{self.tok[0]}": [\n' for child in self.tok[3]: ret += child.indent(0) + ',\n' ret += '\n]' @@ -93,8 +94,10 @@ def indent(self, level=0, idt=' '): SEQ_ELEM_LIST = p.delimitedList(ROOT) SEQUENCE = KW_SEQUENCE + ":" + "[" + p.Group(p.delimitedList(ROOT)) + "]" CHILD = KW_CHILD + ":" + ROOT -CHILD_NODE << (CHILD | COINCIDENT | DOMAIN | EXTENSION | FILTER | MARK | - OPTIONS | PERMUTABLE | SCHEDULE | SEQUENCE) +CHILD_NODE << ( + CHILD | COINCIDENT | DOMAIN | EXTENSION | FILTER | MARK | OPTIONS | + PERMUTABLE | SCHEDULE | SEQUENCE +) ROOT << ("{" + p.Group(p.delimitedList(CHILD_NODE)) + "}") CHILD.addParseAction(Node) @@ -111,7 +114,7 @@ def indent(self, level=0, idt=' '): SEQUENCE.addParseAction(SequenceNode) -def parse_schedule_tree(tree_str): +def parse_schedule_tree(tree_str: tp.Optional[str]) -> tp.Optional[str]: if tree_str is None: return "No Input" diff --git a/benchbuild/utils/settings.py b/benchbuild/utils/settings.py index 8cf0f5411..9cca9b0d7 100644 --- a/benchbuild/utils/settings.py +++ b/benchbuild/utils/settings.py @@ -9,6 +9,8 @@ A leaf node in the dictionary tree is represented by an inner node that contains a value key. """ +from __future__ import annotations + import copy import logging import os @@ -102,7 +104,7 @@ def current_available_threads() -> int: return len(os.sched_getaffinity(0)) -def get_number_of_jobs(config: 'Configuration') -> int: +def get_number_of_jobs(config: Configuration) -> int: """Returns the number of jobs set in the config.""" jobs_configured = int(config["jobs"]) if jobs_configured == 0: @@ -234,7 +236,7 @@ def __init__( self, parent_key: str, node: tp.Optional[InnerNode] = None, - parent: tp.Optional['Configuration'] = None, + parent: tp.Optional[Configuration] = None, init: bool = True ): self.parent = parent @@ -279,7 +281,7 @@ def load(self, _from: LocalPath) -> None: """Load the configuration dictionary from file.""" def load_rec( - inode: tp.Dict[str, tp.Any], config: Configuration + inode: tp.MutableMapping[str, tp.Any], config: Configuration ) -> None: """Recursive part of loading.""" for k in config: @@ -351,7 +353,7 @@ def validate(node_value: tp.Any) -> tp.Any: return validate(self.node['value']) return self - def __getitem__(self, key: str) -> 'Configuration': + def __getitem__(self, key: str) -> Configuration: if key not in self.node: warnings.warn( "Access to non-existing config element: {0}".format(key), @@ -455,7 +457,7 @@ def convert_components(value: tp.Union[str, tp.List[str]]) -> tp.List[str]: @attr.s(str=False, frozen=True) class ConfigPath: """Wrapper around paths represented as list of strings.""" - components = attr.ib(converter=convert_components) + components: tp.List[str] = attr.ib(converter=convert_components) def validate(self) -> None: """Make sure this configuration path exists.""" From 00c71ec1468f293902872bc952c267b8f4e32424 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Fri, 14 Jan 2022 09:31:57 +0100 Subject: [PATCH 03/32] feat(typing): add type annotations for likwid module --- benchbuild/likwid.py | 45 ++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/benchbuild/likwid.py b/benchbuild/likwid.py index 3f49dfc7e..071f41762 100644 --- a/benchbuild/likwid.py +++ b/benchbuild/likwid.py @@ -3,9 +3,10 @@ Extract information from likwid's CSV output. """ +import typing as tp -def fetch_cols(fstream, split_char=','): +def fetch_cols(fstream: tp.IO[str], split_char: str = ',') -> tp.List[str]: """ Fetch columns from likwid's output stream. @@ -20,7 +21,7 @@ def fetch_cols(fstream, split_char=','): return fstream.readline().strip().split(split_char) -def read_struct(fstream): +def read_struct(fstream: tp.IO[str]) -> tp.Optional[tp.Dict[str, tp.List[str]]]: """ Read a likwid struct from the text stream. @@ -33,7 +34,7 @@ def read_struct(fstream): line = fstream.readline().strip() fragments = line.split(",") fragments = [x for x in fragments if x is not None] - partition = dict() + partition = {} if not len(fragments) >= 3: return None @@ -51,7 +52,7 @@ def read_struct(fstream): return struct -def read_table(fstream): +def read_table(fstream: tp.IO[str]) -> tp.Optional[tp.Dict[str, tp.List[str]]]: """ Read a likwid table info from the text stream. @@ -65,7 +66,7 @@ def read_table(fstream): line = fstream.readline().strip() fragments = line.split(",") fragments = [x for x in fragments if x is not None] - partition = dict() + partition = {} if not len(fragments) >= 4: return None @@ -90,7 +91,9 @@ def read_table(fstream): return struct -def read_structs(fstream): +def read_structs( + fstream: tp.IO[str] +) -> tp.Generator[tp.Optional[tp.Dict[str, tp.List[str]]], None, None]: """ Read all structs from likwid's file stream. @@ -107,7 +110,9 @@ def read_structs(fstream): struct = read_struct(fstream) -def read_tables(fstream): +def read_tables( + fstream: tp.IO[str] +) -> tp.Generator[tp.Optional[tp.Dict[str, tp.List[str]]], None, None]: """ Read all tables from likwid's file stream. @@ -123,7 +128,7 @@ def read_tables(fstream): table = read_table(fstream) -def get_measurements(region, core_info, data, extra_offset=0): +def get_measurements(region: str, core_info, data, extra_offset: int = 0): """ Get the complete measurement info from likwid's region info. @@ -152,7 +157,7 @@ def get_measurements(region, core_info, data, extra_offset=0): return measurements -def perfcounters(infile): +def perfcounters(infile: str) -> tp.List[tp.Tuple[str, str, str, str]]: """ Get a complete list of all measurements. @@ -165,21 +170,25 @@ def perfcounters(infile): measurements = [] with open(infile, 'r') as in_file: read_struct(in_file) - for region_struct in read_structs(in_file): + structs = [rs for rs in read_structs(in_file) if rs is not None] + for region_struct in structs: region = region_struct["1"][1] core_info = region_struct["Region Info"] measurements += \ get_measurements(region, core_info, region_struct) - for table_struct in read_tables(in_file): - core_info = None + tables = [ts for ts in read_tables(in_file) if ts is not None] + for table_struct in tables: + evt_core_info: tp.Optional[tp.List[str]] = None if "Event" in table_struct: offset = 1 - core_info = table_struct["Event"][offset:] - measurements += get_measurements(region, core_info, - table_struct, offset) + evt_core_info = table_struct["Event"][offset:] + measurements += get_measurements( + region, evt_core_info, table_struct, offset + ) elif "Metric" in table_struct: - core_info = table_struct["Metric"] - measurements += get_measurements(region, core_info, - table_struct) + evt_core_info = table_struct["Metric"] + measurements += get_measurements( + region, evt_core_info, table_struct + ) return measurements From 8a28919d3814fbc74de2c52a21940599035de644 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Fri, 14 Jan 2022 10:40:01 +0100 Subject: [PATCH 04/32] feat(typing): add type annotations for a few more modules --- benchbuild/container.py | 10 +++++++--- benchbuild/extensions/log.py | 5 +++-- benchbuild/utils/actions.py | 6 +++--- benchbuild/utils/slurm.py | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/benchbuild/container.py b/benchbuild/container.py index 6c3c06a0a..160681a20 100644 --- a/benchbuild/container.py +++ b/benchbuild/container.py @@ -8,6 +8,7 @@ import logging import os import sys +import typing as tp from abc import abstractmethod from plumbum import FG, TF, ProcessExecutionError, cli, local @@ -291,7 +292,10 @@ def output_file(self, _container): """Find and writes the output path of a chroot container.""" p = local.path(_container) if p.exists(): - if not ui.ask("Path '{0}' already exists." " Overwrite?".format(p)): + if not ui.ask( + "Path '{0}' already exists." + " Overwrite?".format(p) + ): sys.exit(0) CFG["container"]["output"] = str(p) @@ -370,13 +374,13 @@ class ContainerCreate(cli.Application): exit the bash and pack up the result. """ - _strategy = BashStrategy() + _strategy: ContainerStrategy = BashStrategy() @cli.switch(["-S", "--strategy"], cli.Set("bash", "polyjit", case_sensitive=False), help="Defines the strategy used to create a new container.", mandatory=False) - def strategy(self, strategy): + def strategy(self, strategy: str): """Select strategy based on key. Args: diff --git a/benchbuild/extensions/log.py b/benchbuild/extensions/log.py index 1115ae391..4e143ab31 100644 --- a/benchbuild/extensions/log.py +++ b/benchbuild/extensions/log.py @@ -1,4 +1,5 @@ import logging +import typing as tp from benchbuild.extensions import base from benchbuild.utils import run @@ -9,9 +10,9 @@ class LogTrackingMixin: """Add log-registering capabilities to extensions.""" - _logs = [] + _logs: tp.MutableSequence[str] = [] - def add_log(self, path): + def add_log(self, path: str): """ Add a log to the tracked list. diff --git a/benchbuild/utils/actions.py b/benchbuild/utils/actions.py index fb8765b98..dee866e5c 100644 --- a/benchbuild/utils/actions.py +++ b/benchbuild/utils/actions.py @@ -67,7 +67,7 @@ def step_has_failed(step_results, error_status=None): def to_step_result( func: DecoratedFunction[StepResultVariants] -) -> StepResultList: +) -> tp.Callable[..., StepResultList]: """Convert a function return to a list of StepResults. All Step subclasses automatically wrap the result of their @@ -83,7 +83,7 @@ def to_step_result( """ @ft.wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args, **kwargs) -> StepResultList: """Wrapper stub.""" res = func(*args, **kwargs) if not res: @@ -112,7 +112,7 @@ def wrapper(self, *args, **kwargs): """Wrapper stub.""" res = func(self, *args, **kwargs) if self.status is not StepResult.UNSET: - res = "[{status}]".format(status=self.status.name) + res + res = f'[{self.status.name}] {res}' return res return wrapper diff --git a/benchbuild/utils/slurm.py b/benchbuild/utils/slurm.py index 2ddfe7220..967544435 100755 --- a/benchbuild/utils/slurm.py +++ b/benchbuild/utils/slurm.py @@ -114,7 +114,7 @@ def __save__( if local.path(template_name).exists(): template_path = local.path(template_name).dirname template_name = local.path(template_name).basename - loader = jinja2.FileSystemLoader(template_path) + loader: jinja2.BaseLoader = jinja2.FileSystemLoader(template_path) else: loader = jinja2.PackageLoader('benchbuild', 'res') From 5ef75a5ffbe9b2f09a78447cefcea610022beda0 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 17 Jan 2022 00:09:23 +0100 Subject: [PATCH 05/32] fix(environments): fix misuse of typed API. --- benchbuild/environments/entrypoints/cli.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/benchbuild/environments/entrypoints/cli.py b/benchbuild/environments/entrypoints/cli.py index 54c432a42..a6db0f4e7 100644 --- a/benchbuild/environments/entrypoints/cli.py +++ b/benchbuild/environments/entrypoints/cli.py @@ -432,8 +432,7 @@ def enumerate_experiments( experiments: ExperimentIndex, projects: ProjectIndex ) -> tp.Generator[experiment.Experiment, None, None]: for exp_class in experiments.values(): - prjs = list(enumerate_projects(experiments, projects)) - yield exp_class(projects=prjs) + yield exp_class(projects=list(projects.values())) def create_experiment_images( @@ -516,7 +515,7 @@ def remove_images( publish = bootstrap.bus() for exp in enumerate_experiments(experiments, projects): - for prj in exp.projects: + for prj in enumerate_projects(experiments, projects): version = make_version_tag(*prj.variant.values()) image_tag = make_image_name( f'{exp.name}/{prj.name}/{prj.group}', version From 5f2d91537f0a06ed442af26d4a217f5f96fa8750 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 17 Jan 2022 00:09:55 +0100 Subject: [PATCH 06/32] refactor(experiment): use format string. --- benchbuild/experiment.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/benchbuild/experiment.py b/benchbuild/experiment.py index 8f783f818..88cd63155 100644 --- a/benchbuild/experiment.py +++ b/benchbuild/experiment.py @@ -181,8 +181,7 @@ def actions(self) -> Actions: actns.Clean(p), actns.MakeBuildDir(p), actns.Echo( - message="Selected {0} with version {1}". - format(p.name, version_str) + message=f'Selected {p.name} with version {version_str}' ), actns.ProjectEnvironment(p), ] From 33fa394b709e2a647bca7f840958247382b76d8b Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 17 Jan 2022 00:31:35 +0100 Subject: [PATCH 07/32] fix(cli/log): fix unreachable type annotations. --- benchbuild/cli/log.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/benchbuild/cli/log.py b/benchbuild/cli/log.py index 4471f204e..6819da632 100644 --- a/benchbuild/cli/log.py +++ b/benchbuild/cli/log.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """ Analyze the BB database. """ +import typing as tp from plumbum import cli @@ -13,9 +14,12 @@ def print_runs(query): return for tup in query: - print(("{0} @ {1} - {2} id: {3} group: {4}".format( - tup.end, tup.experiment_name, tup.project_name, - tup.experiment_group, tup.run_group))) + print(( + "{0} @ {1} - {2} id: {3} group: {4}".format( + tup.end, tup.experiment_name, tup.project_name, + tup.experiment_group, tup.run_group + ) + )) def print_logs(query, types=None): @@ -24,9 +28,12 @@ def print_logs(query, types=None): return for run, log in query: - print(("{0} @ {1} - {2} id: {3} group: {4} status: {5}".format( - run.end, run.experiment_name, run.project_name, - run.experiment_group, run.run_group, log.status))) + print(( + "{0} @ {1} - {2} id: {3} group: {4} status: {5}".format( + run.end, run.experiment_name, run.project_name, + run.experiment_group, run.run_group, log.status + ) + )) print(("command: {0}".format(run.command))) if "stderr" in types: print("StdErr:") @@ -73,10 +80,10 @@ def log_type(self, types): """ Set the output types to print. """ self._types = types - _experiments = None - _experiment_ids = None - _project_ids = None - _types = None + _experiments: tp.Optional[tp.List[str]] = None + _experiment_ids: tp.Optional[tp.List[str]] = None + _project_ids: tp.Optional[tp.List[str]] = None + _types: tp.Optional[tp.List[str]] = None def main(self, *projects): """ Run the log command. """ From 596d4de8d0c2b3c1c051d367cf8871cf4f005113 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 17 Jan 2022 00:33:21 +0100 Subject: [PATCH 08/32] fix(experiment): make ClassVar -> InstanceVar conversion explicit to the type checker --- benchbuild/experiment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchbuild/experiment.py b/benchbuild/experiment.py index 88cd63155..4457e7900 100644 --- a/benchbuild/experiment.py +++ b/benchbuild/experiment.py @@ -107,7 +107,8 @@ def __new__(cls, *args, **kwargs): return new_self name: str = attr.ib( - default=attr.Factory(lambda self: type(self).NAME, takes_self=True) + default=attr. + Factory(lambda self: str(type(self).NAME), takes_self=True) ) projects: Projects = \ From 9c402fbcd1377f71ac879d2877b8fe748d44912b Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 17 Jan 2022 00:33:39 +0100 Subject: [PATCH 09/32] fix(apollo): make ClassVar -> InstanceVar conversion explicit to the type checker --- benchbuild/project.py | 9 ++++++--- benchbuild/projects/apollo/rodinia.py | 23 +++++++++++++++-------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/benchbuild/project.py b/benchbuild/project.py index 5c311807a..a6e5ca8d3 100644 --- a/benchbuild/project.py +++ b/benchbuild/project.py @@ -204,15 +204,18 @@ def __default_variant(self) -> VariantContext: return source.default(*type(self).SOURCE) name: str = attr.ib( - default=attr.Factory(lambda self: type(self).NAME, takes_self=True) + default=attr. + Factory(lambda self: str(type(self).NAME), takes_self=True) ) domain: str = attr.ib( - default=attr.Factory(lambda self: type(self).DOMAIN, takes_self=True) + default=attr. + Factory(lambda self: str(type(self).DOMAIN), takes_self=True) ) group: str = attr.ib( - default=attr.Factory(lambda self: type(self).GROUP, takes_self=True) + default=attr. + Factory(lambda self: str(type(self).GROUP), takes_self=True) ) container: ContainerImage = attr.ib(default=attr.Factory(ContainerImage)) diff --git a/benchbuild/projects/apollo/rodinia.py b/benchbuild/projects/apollo/rodinia.py index c2c74cd79..1bd9f89f6 100644 --- a/benchbuild/projects/apollo/rodinia.py +++ b/benchbuild/projects/apollo/rodinia.py @@ -1,3 +1,5 @@ +import typing as tp + import attr from plumbum import local @@ -12,17 +14,22 @@ class RodiniaGroup(bb.Project): DOMAIN = 'rodinia' GROUP = 'rodinia' SOURCE = [ - HTTP(remote={ - '3.1': 'http://www.cs.virginia.edu/' - '~kw5na/lava/Rodinia/Packages/Current/3.1/' - 'rodinia_3.1.tar.bz2' - }, - local='rodinia.tar.bz2') + HTTP( + remote={ + '3.1': + 'http://www.cs.virginia.edu/' + '~kw5na/lava/Rodinia/Packages/Current/3.1/' + 'rodinia_3.1.tar.bz2' + }, + local='rodinia.tar.bz2' + ) ] - CONFIG = {} + CONFIG: tp.ClassVar[tp.Dict[str, tp.Any]] = {} config = attr.ib( - default=attr.Factory(lambda self: type(self).CONFIG, takes_self=True)) + default=attr. + Factory(lambda self: dict(type(self).CONFIG), takes_self=True) + ) def compile(self): tar('xf', 'rodinia.tar.bz2') From 1595743ce0b5ef5d66d03ebbcf0afbec738e21df Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 17 Jan 2022 00:36:16 +0100 Subject: [PATCH 10/32] fix(versions): remove unreachable code --- benchbuild/utils/versions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/benchbuild/utils/versions.py b/benchbuild/utils/versions.py index e6cb7a474..66e7a18db 100644 --- a/benchbuild/utils/versions.py +++ b/benchbuild/utils/versions.py @@ -29,8 +29,6 @@ def get_version_from_cache_dir(src_file): if tmp_dir.exists(): cache_file = tmp_dir / src_file dir_hash = get_hash_of_dirs(cache_file) - if dir_hash is None: - return None if len(str(dir_hash)) <= 7: return str(dir_hash) return str(dir_hash)[:7] From a3d4d51705bde7201edee009cc590034ea19b76e Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 17 Jan 2022 00:52:52 +0100 Subject: [PATCH 11/32] fix(cli/project): fix type annotations --- benchbuild/cli/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchbuild/cli/project.py b/benchbuild/cli/project.py index d967f2de9..5e502cfde 100644 --- a/benchbuild/cli/project.py +++ b/benchbuild/cli/project.py @@ -74,7 +74,7 @@ def print_project(project: tp.Type[Project], limit: int) -> None: print(f'domain: {project.DOMAIN}') print('source:') for source in project.SOURCE: - num_versions = len(source.versions()) + num_versions = len(list(source.versions())) print(' -', f'{source.remote}') print(' ', 'default:', source.default) From 6aed20040b7e21dc90fda02b8b3de71ef5a09ffd Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 17 Jan 2022 00:53:25 +0100 Subject: [PATCH 12/32] fix(experiment): use future annotations support for type annotations --- benchbuild/experiment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchbuild/experiment.py b/benchbuild/experiment.py index 4457e7900..9cae069fc 100644 --- a/benchbuild/experiment.py +++ b/benchbuild/experiment.py @@ -24,6 +24,8 @@ class HelloExperiment(Experiment): ``` """ +from __future__ import annotations + import copy import typing as tp import uuid @@ -48,7 +50,7 @@ class HelloExperiment(Experiment): class ExperimentRegistry(type): """Registry for benchbuild experiments.""" - experiments = {} + experiments: ExperimentIndex = {} def __new__( mcs: tp.Type[tp.Any], name: str, bases: tp.Tuple[type, ...], From 6fcabc92ff8ff1953b5b16dbb3d2214090fe3c43 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 17 Jan 2022 00:53:56 +0100 Subject: [PATCH 13/32] refactor(utils/actions): use format strings where applicable --- benchbuild/utils/actions.py | 42 +++++++++++++++---------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/benchbuild/utils/actions.py b/benchbuild/utils/actions.py index dee866e5c..6c588f0e5 100644 --- a/benchbuild/utils/actions.py +++ b/benchbuild/utils/actions.py @@ -260,8 +260,7 @@ def __call__(self) -> StepResultVariants: def __str__(self, indent: int = 0) -> str: return textwrap.indent( - "* {name}: Execute configured action.".format(name=self.obj.name), - indent * " " + f'* {self.obj.name}: Execute configured action.', indent * " " ) def onerror(self): @@ -317,9 +316,8 @@ def __call__(self) -> StepResultVariants: def __str__(self, indent: int = 0) -> str: return textwrap.indent( - "* {0}: Clean the directory: {1}".format( - self.obj.name, self.obj.builddir - ), indent * " " + f'* {self.obj.name}: Clean the directory: {self.obj.builddir}', + indent * " " ) @@ -338,8 +336,7 @@ def __call__(self) -> StepResultVariants: def __str__(self, indent: int = 0) -> str: return textwrap.indent( - "* {0}: Create the build directory".format(self.obj.name), - indent * " " + f'* {self.obj.name}: Create the build directory', indent * " " ) @@ -351,9 +348,7 @@ def __init__(self, project): super().__init__(obj=project, action_fn=project.compile) def __str__(self, indent: int = 0) -> str: - return textwrap.indent( - "* {0}: Compile".format(self.obj.name), indent * " " - ) + return textwrap.indent(f'* {self.obj.name}: Compile', indent * " ") @attr.s @@ -386,8 +381,7 @@ def __call__(self): def __str__(self, indent: int = 0) -> str: return textwrap.indent( - "* {0}: Execute run-time tests.".format(self.project.name), - indent * " " + f'* {self.project.name}: Execute run-time tests.', indent * " " ) @@ -399,7 +393,7 @@ class Echo(Step): message = attr.ib(default="") def __str__(self, indent: int = 0) -> str: - return textwrap.indent("* echo: {0}".format(self.message), indent * " ") + return textwrap.indent(f'* echo: {self.message}', indent * " ") @notify_step_begin_end def __call__(self) -> StepResultVariants: @@ -422,7 +416,9 @@ class Any(Step): NAME = "ANY" DESCRIPTION = "Just run all actions, no questions asked." - actions = attr.ib(default=attr.Factory(list), repr=False, eq=False) + actions: tp.List[Step] = attr.ib( + default=attr.Factory(list), repr=False, eq=False + ) def __len__(self): return sum([len(x) for x in self.actions]) + 1 @@ -465,9 +461,9 @@ def __attrs_post_init__(self): return self.actions = \ - [Echo(message="Start experiment: {0}".format(self.obj.name))] + \ + [Echo(message=f'Start experiment: {self.obj.name}')] + \ self.actions + \ - [Echo(message="Completed experiment: {0}".format(self.obj.name))] + [Echo(message=f'Completed experiment: {self.obj.name}')] def begin_transaction(self): experiment, session = db.persist_experiment(self.obj) @@ -540,8 +536,7 @@ def __str__(self, indent: int = 0) -> str: sub_actns = [a.__str__(indent + 1) for a in self.actions] sub_actns_str = "\n".join(sub_actns) return textwrap.indent( - "\nExperiment: {0}\n".format(self.obj.name) + sub_actns_str, - indent * " " + f'\nExperiment: {self.obj.name}\n' + sub_actns_str, indent * " " ) @@ -550,7 +545,7 @@ class RequireAll(Step): NAME = "REQUIRE ALL" DESCRIPTION = "All child steps need to succeed" - actions = attr.ib(default=attr.Factory(list)) + actions: tp.List[Step] = attr.ib(default=attr.Factory(list)) def __len__(self) -> int: return sum([len(x) for x in self.actions]) + 1 @@ -665,9 +660,7 @@ def __str__(self, indent: int = 0) -> str: lines = [] for p in paths: lines.append( - textwrap.indent( - "* Clean the directory: {0}".format(p), indent * " " - ) + textwrap.indent(f'* Clean the directory: {p}', indent * " ") ) return "\n".join(lines) @@ -692,7 +685,6 @@ def __str__(self, indent: int = 0) -> str: version_str = source.to_str(*tuple(variant.values())) return textwrap.indent( - "* Project environment for: {} @ {}".format( - project.name, version_str - ), indent * " " + f'* Project environment for: {project.name} @ {version_str}', + indent * " " ) From 85c3cff81aa29e7daa8d51d7e9a92434cd08d93c Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 17 Jan 2022 01:35:48 +0100 Subject: [PATCH 14/32] refactor: convert .format() to format strings --- benchbuild/cli/bootstrap.py | 3 +- benchbuild/cli/config.py | 3 +- benchbuild/cli/log.py | 20 +++++------- benchbuild/cli/main.py | 2 +- benchbuild/cli/project.py | 4 +-- benchbuild/cli/run.py | 10 +++--- benchbuild/container.py | 37 ++++++++++++----------- benchbuild/engine.py | 2 +- benchbuild/experiment.py | 4 +-- benchbuild/projects/benchbuild/bots.py | 8 +++-- benchbuild/projects/benchbuild/ffmpeg.py | 3 +- benchbuild/projects/benchbuild/gzip.py | 2 +- benchbuild/projects/benchbuild/lapack.py | 4 +-- benchbuild/projects/benchbuild/leveldb.py | 7 ++--- benchbuild/projects/benchbuild/povray.py | 4 +-- benchbuild/projects/benchbuild/sqlite3.py | 4 +-- benchbuild/projects/benchbuild/tcc.py | 2 +- benchbuild/projects/gentoo/__init__.py | 3 +- benchbuild/projects/gentoo/gentoo.py | 30 +++++++++--------- benchbuild/projects/gentoo/info.py | 2 +- benchbuild/source/http.py | 2 +- benchbuild/utils/bootstrap.py | 14 ++++----- benchbuild/utils/compiler.py | 6 ++-- benchbuild/utils/container.py | 21 +++++++------ benchbuild/utils/path.py | 12 +++++--- benchbuild/utils/schema.py | 19 +++++------- benchbuild/utils/settings.py | 2 +- benchbuild/utils/slurm.py | 2 +- benchbuild/utils/uchroot.py | 6 ++-- benchbuild/utils/user_interface.py | 2 +- benchbuild/utils/wrapping.py | 4 +-- 31 files changed, 116 insertions(+), 128 deletions(-) diff --git a/benchbuild/cli/bootstrap.py b/benchbuild/cli/bootstrap.py index 4ba199031..3dc8a83b1 100644 --- a/benchbuild/cli/bootstrap.py +++ b/benchbuild/cli/bootstrap.py @@ -30,5 +30,6 @@ def main(self, *args: str) -> int: if self.store_config: config_path = ".benchbuild.yml" CFG.store(config_path) - print("Storing config in {0}".format(os.path.abspath(config_path))) + store_path = os.path.abspath(config_path) + print(f'Storing config in {store_path}') return 0 diff --git a/benchbuild/cli/config.py b/benchbuild/cli/config.py index ffd92f66a..b09abd64b 100644 --- a/benchbuild/cli/config.py +++ b/benchbuild/cli/config.py @@ -31,4 +31,5 @@ class BBConfigWrite(cli.Application): def main(self): config_path = ".benchbuild.yml" settings.CFG.store(config_path) - print("Storing config in {0}".format(os.path.abspath(config_path))) + store_path = os.path.abspath(config_path) + print(f'Storing config in {store_path}') diff --git a/benchbuild/cli/log.py b/benchbuild/cli/log.py index 6819da632..f80681da6 100644 --- a/benchbuild/cli/log.py +++ b/benchbuild/cli/log.py @@ -14,12 +14,9 @@ def print_runs(query): return for tup in query: - print(( - "{0} @ {1} - {2} id: {3} group: {4}".format( - tup.end, tup.experiment_name, tup.project_name, - tup.experiment_group, tup.run_group - ) - )) + print( + f'{tup.end} @ {tup.experiment_name} - {tup.project_name} id: {tup.experiment_group} group: {tup.run_group}' + ) def print_logs(query, types=None): @@ -28,13 +25,10 @@ def print_logs(query, types=None): return for run, log in query: - print(( - "{0} @ {1} - {2} id: {3} group: {4} status: {5}".format( - run.end, run.experiment_name, run.project_name, - run.experiment_group, run.run_group, log.status - ) - )) - print(("command: {0}".format(run.command))) + print( + f'{run.end} @ {run.experiment_name} - {run.project_name} id: {run.experiment_group} group: {run.run_group} status: {log.status}' + ) + print(f'command: {run.command}') if "stderr" in types: print("StdErr:") print((log.stderr)) diff --git a/benchbuild/cli/main.py b/benchbuild/cli/main.py index de0cf9028..fbcfd4e59 100644 --- a/benchbuild/cli/main.py +++ b/benchbuild/cli/main.py @@ -44,7 +44,7 @@ def main(self, *args: str) -> int: init_functions(Session()) if args: - print("Unknown command {0!r}".format(args[0])) + print(f'Unknown command {args[0]!r}') return 1 if not self.nested_command: self.help() diff --git a/benchbuild/cli/project.py b/benchbuild/cli/project.py index 5e502cfde..36f04e650 100644 --- a/benchbuild/cli/project.py +++ b/benchbuild/cli/project.py @@ -113,9 +113,7 @@ def print_projects(projects: ProjectIndex) -> None: if prj.GROUP not in grouped_by: grouped_by[prj.GROUP] = [] - grouped_by[prj.GROUP].append( - "{name}/{group}".format(name=prj.NAME, group=prj.GROUP) - ) + grouped_by[prj.GROUP].append(f'{prj.NAME}/{prj.GROUP}') project_column_width = max([ len(f'{p.NAME}/{p.GROUP}') for p in projects.values() diff --git a/benchbuild/cli/run.py b/benchbuild/cli/run.py index a71d8b800..945cca974 100644 --- a/benchbuild/cli/run.py +++ b/benchbuild/cli/run.py @@ -116,13 +116,11 @@ def print_summary(num_actions, failed, duration): """ num_failed = len(failed) print( - """ + f''' Summary: -{num_total} actions were in the queue. +{num_actions} actions were in the queue. {num_failed} actions failed to execute. -This run took: {elapsed_time:8.3f} seconds. - """.format( - num_total=num_actions, num_failed=num_failed, elapsed_time=duration - ) +This run took: {duration:8.3f} seconds. + ''' ) diff --git a/benchbuild/container.py b/benchbuild/container.py index 160681a20..1f87cac6a 100644 --- a/benchbuild/container.py +++ b/benchbuild/container.py @@ -28,10 +28,10 @@ def clean_directories(builddir, in_dir=True, out_dir=True): container_out = local.path(builddir) / "container-out" if in_dir and container_in.exists(): - if ui.ask("Should I delete '{0}'?".format(container_in)): + if ui.ask(f'Should I delete \'{container_in}\'?'): container_in.delete() if out_dir and container_out.exists(): - if ui.ask("Should I delete '{0}'?".format(container_out)): + if ui.ask(f'Should I delete \'{container_out}\'?'): container_out.delete() @@ -64,9 +64,9 @@ def setup_container(builddir, _container): os.path.abspath("."), "--"] # Check, if we need erlent support for this archive. - has_erlent = bash["-c", - "tar --list -f './{0}' | grep --silent '.erlent'". - format(container_in)] + has_erlent = bash[ + "-c", + f'tar --list -f \'./{container_in}\' | grep --silent \'.erlent\''] has_erlent = (has_erlent & TF) # Unpack input container to: container-in @@ -158,7 +158,8 @@ def setup_bash_in_container(builddir, _container, outfile, shell): pack_container(_container, outfile) config_path = str(CFG["config_file"]) CFG.store(config_path) - print("Storing config in {0}".format(os.path.abspath(config_path))) + store_path = os.path.abspath(config_path) + print(f'Storing config in {store_path}') def find_hash(container_db, key): @@ -245,7 +246,8 @@ def run(self, context): packages = \ CFG["container"]["strategy"]["polyjit"]["packages"].value - with local.env(MAKEOPTS="-j{0}".format(get_number_of_jobs(CFG))): + num_jobs = get_number_of_jobs(CFG) + with local.env(MAKEOPTS=f'-j{num_jobs}'): if want_sync: LOG.debug("Synchronizing portage.") emerge_in_chroot("--sync") @@ -285,17 +287,15 @@ def input_file(self, _container): if set_input_container(p, CFG): return - raise ValueError("The path '{0}' does not exist.".format(p)) + raise ValueError(f'The path \'{p}\' does not exist.') @cli.switch(["-o", "--output-file"], str, help="Output container path") def output_file(self, _container): """Find and writes the output path of a chroot container.""" p = local.path(_container) if p.exists(): - if not ui.ask( - "Path '{0}' already exists." - " Overwrite?".format(p) - ): + if not ui.ask(f'Path \'{p}\' already exists.' + " Overwrite?"): sys.exit(0) CFG["container"]["output"] = str(p) @@ -330,13 +330,13 @@ def main(self, *args): builddir = local.path(str(CFG["build_dir"])) if not builddir.exists(): response = ui.ask( - "The build directory {dirname} does not exist yet. " - "Should I create it?".format(dirname=builddir) + f'The build directory {builddir} does not exist yet. ' + "Should I create it?" ) if response: mkdir("-p", builddir) - print("Created directory {0}.".format(builddir)) + print(f'Created directory {builddir}.') setup_directories(builddir) @@ -443,7 +443,8 @@ def main(self, *args): if not (config_file and os.path.exists(config_file)): config_file = ".benchbuild.json" CFG.store(config_file) - print("Storing config in {0}".format(os.path.abspath(config_file))) + store_path = os.path.abspath(config_file) + print(f'Storing config in {store_path}') print( "Future container commands from this directory will automatically" " source the config file." @@ -457,7 +458,9 @@ class ContainerList(cli.Application): def main(self, *args): containers = CFG["container"]["known"].value for c in containers: - print("[{1:.8s}] {0}".format(c["path"], str(c["hash"]))) + c_path = c["path"] + c_hash = str(c["hash"]) + print(f'[{c_hash:.8s}] {c_path}') def main(*args): diff --git a/benchbuild/engine.py b/benchbuild/engine.py index f2f3e2812..96b78d94f 100644 --- a/benchbuild/engine.py +++ b/benchbuild/engine.py @@ -43,5 +43,5 @@ def start(self) -> StepResults: def print_plan(self) -> None: p = self.plan() - print("Number of actions to execute: {}".format(self.num_actions)) + print(f'Number of actions to execute: {self.num_actions}') print(*p) diff --git a/benchbuild/experiment.py b/benchbuild/experiment.py index 9cae069fc..c75283884 100644 --- a/benchbuild/experiment.py +++ b/benchbuild/experiment.py @@ -102,9 +102,7 @@ def __new__(cls, *args, **kwargs): new_self = super(Experiment, cls).__new__(cls) if not cls.NAME: raise AttributeError( - "{0} @ {1} does not define a NAME class attribute.".format( - cls.__name__, cls.__module__ - ) + f'{cls.__name__} @ {cls.__module__} does not define a NAME class attribute.' ) return new_self diff --git a/benchbuild/projects/benchbuild/bots.py b/benchbuild/projects/benchbuild/bots.py index cedcf880e..65a9477bf 100644 --- a/benchbuild/projects/benchbuild/bots.py +++ b/benchbuild/projects/benchbuild/bots.py @@ -27,10 +27,12 @@ class BOTSGroup(bb.Project): DOMAIN = 'bots' GROUP = 'bots' SOURCE = [ - Git(remote='https://github.com/bsc-pm/bots', + Git( + remote='https://github.com/bsc-pm/bots', local='bots.git', limit=5, - refspec='HEAD') + refspec='HEAD' + ) ] path_dict = { @@ -96,7 +98,7 @@ def compile(self): _make("-C", self.path_dict[self.name]) def run_tests(self): - binary_name = "{name}.benchbuild.serial".format(name=self.name) + binary_name = f'{self.name}.benchbuild.serial' bots_repo = local.path(self.source_of('bots.git')) binary_path = bots_repo / "bin" / binary_name exp = bb.wrap(binary_path, self) diff --git a/benchbuild/projects/benchbuild/ffmpeg.py b/benchbuild/projects/benchbuild/ffmpeg.py index 1f8207a25..b0325688b 100644 --- a/benchbuild/projects/benchbuild/ffmpeg.py +++ b/benchbuild/projects/benchbuild/ffmpeg.py @@ -52,4 +52,5 @@ def compile(self): "--samples=" + self.fate_dir ) _make("clean") - _make("-j{0}".format(str(get_number_of_jobs(CFG))), "all") + num_jobs = str(get_number_of_jobs(CFG)) + _make(f'-j{num_jobs}', "all") diff --git a/benchbuild/projects/benchbuild/gzip.py b/benchbuild/projects/benchbuild/gzip.py index 9765de3da..3074d3150 100644 --- a/benchbuild/projects/benchbuild/gzip.py +++ b/benchbuild/projects/benchbuild/gzip.py @@ -55,7 +55,7 @@ def compile(self): tar('xf', compression_source) gzip_version = self.version_of('gzip.tar.xz') - unpack_dir = "gzip-{0}.tar.xz".format(gzip_version) + unpack_dir = f'gzip-{gzip_version}.tar.xz' clang = bb.compiler.cc(self) with local.cwd(unpack_dir): diff --git a/benchbuild/projects/benchbuild/lapack.py b/benchbuild/projects/benchbuild/lapack.py index c4b675648..ebdf05afa 100644 --- a/benchbuild/projects/benchbuild/lapack.py +++ b/benchbuild/projects/benchbuild/lapack.py @@ -53,7 +53,7 @@ def compile(self): clapack_version = self.version_of('clapack.tgz') tar("xfz", clapack_source) - unpack_dir = "CLAPACK-{0}".format(clapack_version) + unpack_dir = f'CLAPACK-{clapack_version}' clang = bb.compiler.cc(self) clang_cxx = bb.compiler.cxx(self) @@ -86,7 +86,7 @@ def compile(self): def run_tests(self): clapack_version = self.version_of('clapack.tgz') - unpack_dir = local.path("CLAPACK-{0}".format(clapack_version)) + unpack_dir = local.path(f'CLAPACK-{clapack_version}') with local.cwd(unpack_dir / "BLAS"): xblat2s = bb.wrap("xblat2s", self) _xblat2s = bb.watch((xblat2s < "sblat2.in")) diff --git a/benchbuild/projects/benchbuild/leveldb.py b/benchbuild/projects/benchbuild/leveldb.py index 4559ae6eb..9a64b01f5 100644 --- a/benchbuild/projects/benchbuild/leveldb.py +++ b/benchbuild/projects/benchbuild/leveldb.py @@ -45,8 +45,7 @@ def run_tests(self): leveldb = bb.wrap(leveldb_repo / "out-static" / "db_bench", self) _leveldb = bb.watch(leveldb) - with local.env( - LD_LIBRARY_PATH="{}:{}". - format(leveldb_repo / "out-shared", getenv("LD_LIBRARY_PATH", "")) - ): + out_shared = leveldb_repo / "out-shared" + lib_path = getenv("LD_LIBRARY_PATH", "") + with local.env(LD_LIBRARY_PATH=f'{out_shared}:{lib_path}'): _leveldb() diff --git a/benchbuild/projects/benchbuild/povray.py b/benchbuild/projects/benchbuild/povray.py index a8ff6f5c0..f86a7d1e3 100644 --- a/benchbuild/projects/benchbuild/povray.py +++ b/benchbuild/projects/benchbuild/povray.py @@ -62,9 +62,7 @@ def compile(self): mkdir(boost_prefix) bootstrap = local["./bootstrap.sh"] _bootstrap = bb.watch(bootstrap) - _bootstrap( - "--with-toolset=clang", "--prefix=\"{0}\"".format(boost_prefix) - ) + _bootstrap("--with-toolset=clang", f'--prefix="{boost_prefix}"') _b2 = bb.watch(local["./b2"]) _b2( diff --git a/benchbuild/projects/benchbuild/sqlite3.py b/benchbuild/projects/benchbuild/sqlite3.py index bb242b3d6..e20581e04 100644 --- a/benchbuild/projects/benchbuild/sqlite3.py +++ b/benchbuild/projects/benchbuild/sqlite3.py @@ -51,8 +51,8 @@ def build_leveldb(self): leveldb_repo = local.path(self.source_of('leveldb.src')) # We need to place sqlite3 in front of all other flags. - self.ldflags += ["-L{0}".format(sqlite_dir)] - self.cflags += ["-I{0}".format(sqlite_dir)] + self.ldflags += [f'-L{sqlite_dir}'] + self.cflags += [f'-I{sqlite_dir}'] clang_cxx = bb.compiler.cxx(self) clang = bb.compiler.cc(self) diff --git a/benchbuild/projects/benchbuild/tcc.py b/benchbuild/projects/benchbuild/tcc.py index fe0870ed5..c9d41b287 100644 --- a/benchbuild/projects/benchbuild/tcc.py +++ b/benchbuild/projects/benchbuild/tcc.py @@ -53,4 +53,4 @@ def run_tests(self): bb.wrap("tcc", self) inc_path = path.abspath("..") _make = bb.watch(make) - _make("TCCFLAGS=-B{}".format(inc_path), "test", "-i") + _make(f'TCCFLAGS=-B{inc_path}', "test", "-i") diff --git a/benchbuild/projects/gentoo/__init__.py b/benchbuild/projects/gentoo/__init__.py index e510a1211..628c048a7 100644 --- a/benchbuild/projects/gentoo/__init__.py +++ b/benchbuild/projects/gentoo/__init__.py @@ -40,8 +40,7 @@ def __initialize_dynamic_projects__(autotest_path): domain = ebuild_data_list[0] name = ebuild_data_list[1] PortageFactory( - "Auto{0}{1}".format(domain, name), domain + "_" + name, - domain + f'Auto{domain}{name}', domain + "_" + name, domain ) diff --git a/benchbuild/projects/gentoo/gentoo.py b/benchbuild/projects/gentoo/gentoo.py index 3cc3322a9..8ffe1f69f 100644 --- a/benchbuild/projects/gentoo/gentoo.py +++ b/benchbuild/projects/gentoo/gentoo.py @@ -53,13 +53,11 @@ def redirect(self) -> None: benchbuild = find_benchbuild() _benchbuild = run.watch(benchbuild) with local.env(BB_VERBOSITY=str(CFG['verbosity'])): - project_id = "{0}/{1}".format(self.name, self.group) + project_id = f'{self.name}/{self.group}' _benchbuild("run", "-E", self.experiment.name, project_id) def compile(self) -> None: - package_atom = "{domain}/{name}".format( - domain=self.domain, name=self.name - ) + package_atom = f'{self.domain}/{self.name}' LOG.debug('Installing dependencies.') emerge(package_atom, '--onlydeps', **self.emerge_env) @@ -178,17 +176,17 @@ def write_makeconfig(_path: str) -> None: CFG["container"]["mounts"] = mounts if http_proxy is not None: - http_s = "http_proxy={0}".format(http_proxy) - https_s = "https_proxy={0}".format(http_proxy) + http_s = f'http_proxy={http_proxy}' + https_s = f'https_proxy={http_proxy}' makeconf.write(http_s + "\n") makeconf.write(https_s + "\n") if ftp_proxy is not None: - fp_s = "ftp_proxy={0}".format(ftp_proxy) + fp_s = f'ftp_proxy={ftp_proxy}' makeconf.write(fp_s + "\n") if rsync_proxy is not None: - rp_s = "RSYNC_PROXY={0}".format(rsync_proxy) + rp_s = f'RSYNC_PROXY={rsync_proxy}' makeconf.write(rp_s + "\n") @@ -211,10 +209,12 @@ def write_bashrc(_path: str) -> None: libs = libs + p_libs with open(_path, 'w') as bashrc: - lines = ''' -export PATH="{0}:${{PATH}}" -export LD_LIBRARY_PATH="{1}:${{LD_LIBRARY_PATH}}" -'''.format(path.list_to_path(paths), path.list_to_path(libs)) + path_var = path.list_to_path(paths) + lib_path_var = path.list_to_path(libs) + lines = f''' +export PATH="{path_var}:${{PATH}}" +export LD_LIBRARY_PATH="{lib_path_var}:${{LD_LIBRARY_PATH}}" +''' bashrc.write(lines) @@ -247,14 +247,14 @@ def write_wgetrc(_path: str) -> None: uchroot.mkfile_uchroot("/etc/wgetrc") with open(_path, 'w') as wgetrc: if http_proxy is not None: - http_s = "http_proxy = {0}".format(http_proxy) - https_s = "https_proxy = {0}".format(http_proxy) + http_s = f'http_proxy = {http_proxy}' + https_s = f'https_proxy = {http_proxy}' wgetrc.write("use_proxy = on\n") wgetrc.write(http_s + "\n") wgetrc.write(https_s + "\n") if ftp_proxy is not None: - fp_s = "ftp_proxy={0}".format(ftp_proxy) + fp_s = f'ftp_proxy={ftp_proxy}' wgetrc.write(fp_s + "\n") diff --git a/benchbuild/projects/gentoo/info.py b/benchbuild/projects/gentoo/info.py index 73bad2019..fc4f66f36 100644 --- a/benchbuild/projects/gentoo/info.py +++ b/benchbuild/projects/gentoo/info.py @@ -40,7 +40,7 @@ def compile(self): for line in output.split('\n'): if "ebuild" in line: parts = line.split('.ebuild')[0].split('/') - package_atom = '{0}/{1}'.format(parts[0], parts[1]) + package_atom = f'{parts[0]}/{parts[1]}' ebuilds.add(package_atom) for use in use_flags: diff --git a/benchbuild/source/http.py b/benchbuild/source/http.py index c4b3190b1..0cce2c314 100644 --- a/benchbuild/source/http.py +++ b/benchbuild/source/http.py @@ -52,7 +52,7 @@ def normalize_remotes(remote: VarRemotes) -> Remotes: def versioned_target_name(target_name: str, version: str) -> str: - return "{}-{}".format(version, target_name) + return f'{version}-{target_name}' def download_single_version(url: str, target_path: str) -> str: diff --git a/benchbuild/utils/bootstrap.py b/benchbuild/utils/bootstrap.py index 5a06aa92b..500dd02cb 100644 --- a/benchbuild/utils/bootstrap.py +++ b/benchbuild/utils/bootstrap.py @@ -31,9 +31,9 @@ def find_package(binary: str) -> bool: found = not isinstance(c, utils.ErrorCommand) if found: - print("Checking for {} - Yes [{}]".format(binary, str(c))) + print(f'Checking for {binary} - Yes [{str(c)}]') else: - print("Checking for {} - No".format(binary)) + print(f'Checking for {binary} - No') return found @@ -137,15 +137,13 @@ def check_uchroot_config() -> None: if not (fuse_grep["^user_allow_other", "/etc/fuse.conf"] & TF): print("uchroot needs 'user_allow_other' enabled in '/etc/fuse.conf'.") - if not (fuse_grep["^{0}".format(username), "/etc/subuid"] & TF): + if not (fuse_grep[f'^{username}', "/etc/subuid"] & TF): print( - "uchroot needs an entry for user '{0}' in '/etc/subuid'.". - format(username) + f'uchroot needs an entry for user \'{username}\' in \'/etc/subuid\'.' ) - if not (fuse_grep["^{0}".format(username), "/etc/subgid"] & TF): + if not (fuse_grep[f'^{username}', "/etc/subgid"] & TF): print( - "uchroot needs an entry for user '{0}' in '/etc/subgid'.". - format(username) + f'uchroot needs an entry for user \'{username}\' in \'/etc/subgid\'.' ) diff --git a/benchbuild/utils/compiler.py b/benchbuild/utils/compiler.py index 76874200c..05303e1dd 100644 --- a/benchbuild/utils/compiler.py +++ b/benchbuild/utils/compiler.py @@ -31,8 +31,8 @@ from benchbuild.utils.wrapping import wrap_cc if TYPE_CHECKING: - from benchbuild.project import Project from benchbuild.experiment import Experiment + from benchbuild.project import Project def cc(project: 'Project', detect_project: bool = False) -> BoundCommand: @@ -55,7 +55,7 @@ def cc(project: 'Project', detect_project: bool = False) -> BoundCommand: """ cc_name = str(CFG["compiler"]["c"]) wrap_cc(cc_name, compiler(cc_name), project, detect_project=detect_project) - return cmd["./{}".format(cc_name)] + return cmd[f'./{cc_name}'] def cxx(project: 'Project', detect_project: bool = False) -> BoundCommand: @@ -81,7 +81,7 @@ def cxx(project: 'Project', detect_project: bool = False) -> BoundCommand: wrap_cc( cxx_name, compiler(cxx_name), project, detect_project=detect_project ) - return cmd["./{name}".format(name=cxx_name)] + return cmd[f'./{cxx_name}'] def __get_paths() -> tp.Dict[str, str]: diff --git a/benchbuild/utils/container.py b/benchbuild/utils/container.py index fae14df0d..b9820fb93 100644 --- a/benchbuild/utils/container.py +++ b/benchbuild/utils/container.py @@ -77,12 +77,14 @@ def src_file(self): Latest src_uri from gentoo's distfiles mirror. """ try: - src_uri = (curl[Gentoo._LATEST_TXT] | tail["-n", "+3"] | - cut["-f1", "-d "])().strip() + src_uri = ( + curl[Gentoo._LATEST_TXT] | tail["-n", "+3"] | cut["-f1", "-d "] + )().strip() except ProcessExecutionError as proc_ex: src_uri = "NOT-FOUND" - LOG.error("Could not determine latest stage3 src uri: %s", - str(proc_ex)) + LOG.error( + "Could not determine latest stage3 src uri: %s", str(proc_ex) + ) return src_uri @property @@ -103,8 +105,7 @@ def version(self): @property def remote(self): """Get a remote URL of the requested container.""" - return "http://distfiles.gentoo.org/releases/amd64/autobuilds/{0}" \ - .format(self.src_file) + return f'http://distfiles.gentoo.org/releases/amd64/autobuilds/{self.src_file}' def is_valid(container, path): @@ -164,8 +165,7 @@ def unpack(container, path): # Check, if we need erlent support for this archive. has_erlent = bash[ - "-c", - "tar --list -f './{0}' | grep --silent '.erlent'".format(name)] + "-c", f'tar --list -f \'./{name}\' | grep --silent \'.erlent\''] has_erlent = (has_erlent & TF) untar = local["/bin/tar"]["xf", "./" + name] @@ -177,8 +177,9 @@ def unpack(container, path): if not os.path.samefile(name, container.filename): rm(name) else: - LOG.warning("File contents do not match: %s != %s", name, - container.filename) + LOG.warning( + "File contents do not match: %s != %s", name, container.filename + ) cp(container.filename + ".hash", path) diff --git a/benchbuild/utils/path.py b/benchbuild/utils/path.py index e91302bdf..069f82e62 100644 --- a/benchbuild/utils/path.py +++ b/benchbuild/utils/path.py @@ -116,14 +116,16 @@ def mkdir_interactive(dirpath: str) -> None: if os.path.exists(dirpath): return - response = ui.ask("The directory {dirname} does not exist yet. " - "Should I create it?".format(dirname=dirpath), - default_answer=True, - default_answer_str="yes") + response = ui.ask( + f'The directory {dirpath} does not exist yet. ' + "Should I create it?", + default_answer=True, + default_answer_str="yes" + ) if response: mkdir("-p", dirpath) - print("Created directory {0}.".format(dirpath)) + print(f'Created directory {dirpath}.') @contextmanager diff --git a/benchbuild/utils/schema.py b/benchbuild/utils/schema.py index 17ddd422d..63684e075 100644 --- a/benchbuild/utils/schema.py +++ b/benchbuild/utils/schema.py @@ -184,8 +184,9 @@ class Run(BASE): ) def __repr__(self): - return ("" - ).format(self.project_name, self.status, self.id) + return ( + f'' + ) class RunGroup(BASE): @@ -230,7 +231,7 @@ class Experiment(BASE): ) def __repr__(self): - return "".format(name=self.name) + return f'' class Project(BASE): @@ -253,12 +254,7 @@ class Project(BASE): ) def __repr__(self): - return "".format( - group=self.group_name, - domain=self.domain, - name=self.name, - version=self.version - ) + return f'' class Metric(BASE): @@ -276,7 +272,7 @@ class Metric(BASE): ) def __repr__(self): - return "{0} - {1}".format(self.name, self.value) + return f'{self.name} - {self.value}' class RunLog(BASE): @@ -431,8 +427,7 @@ def maybe_update_db(repo_version, db_version): repo_version ) if not ui.ask( - "Should I attempt to update your schema to version '{0}'?". - format(repo_version) + f'Should I attempt to update your schema to version \'{repo_version}\'?' ): LOG.error("User declined schema upgrade.") return diff --git a/benchbuild/utils/settings.py b/benchbuild/utils/settings.py index 9cca9b0d7..12857547a 100644 --- a/benchbuild/utils/settings.py +++ b/benchbuild/utils/settings.py @@ -356,7 +356,7 @@ def validate(node_value: tp.Any) -> tp.Any: def __getitem__(self, key: str) -> Configuration: if key not in self.node: warnings.warn( - "Access to non-existing config element: {0}".format(key), + f'Access to non-existing config element: {key}', category=InvalidConfigKey, stacklevel=2 ) diff --git a/benchbuild/utils/slurm.py b/benchbuild/utils/slurm.py index 967544435..35abb1384 100755 --- a/benchbuild/utils/slurm.py +++ b/benchbuild/utils/slurm.py @@ -168,7 +168,7 @@ def __save__( chmod("+x", script_name) if not __verify__(script_name): LOG.error("SLURM script failed verification.") - print("SLURM script written to {0}".format(script_name)) + print(f'SLURM script written to {script_name}') return script_name diff --git a/benchbuild/utils/uchroot.py b/benchbuild/utils/uchroot.py index 5e3f6e091..83db50219 100644 --- a/benchbuild/utils/uchroot.py +++ b/benchbuild/utils/uchroot.py @@ -157,7 +157,7 @@ def mounts(prefix: str, __mounts: tp.List) -> tp.List[str]: mntpoints = [] for mount in __mounts: if not isinstance(mount, dict): - mntpoint = "{0}/{1}".format(prefix, str(i)) + mntpoint = f'{prefix}/{str(i)}' mntpoints.append(mntpoint) i = i + 1 return mntpoints @@ -174,10 +174,10 @@ def __mounts__(prefix: str, tgt_mount = mount["tgt"] else: src_mount = mount - tgt_mount = "{0}/{1}".format(prefix, str(i)) + tgt_mount = f'{prefix}/{str(i)}' i = i + 1 mkdir_uchroot(tgt_mount) - uchroot_opts.extend(["-M", "{0}:{1}".format(src_mount, tgt_mount)]) + uchroot_opts.extend(["-M", f'{src_mount}:{tgt_mount}']) mntpoints.append(tgt_mount) return uchroot_opts, mntpoints diff --git a/benchbuild/utils/user_interface.py b/benchbuild/utils/user_interface.py index 370cce569..e153f3ba7 100644 --- a/benchbuild/utils/user_interface.py +++ b/benchbuild/utils/user_interface.py @@ -30,7 +30,7 @@ def query_yes_no(question, default="yes"): elif default == "no": prompt = " [y/N] " else: - raise ValueError("invalid default answer: '{0!s}'".format(default)) + raise ValueError(f'invalid default answer: \'{default!s}\'') while True: sys.stdout.write(question + prompt) diff --git a/benchbuild/utils/wrapping.py b/benchbuild/utils/wrapping.py index 2867206c1..072e70037 100644 --- a/benchbuild/utils/wrapping.py +++ b/benchbuild/utils/wrapping.py @@ -45,8 +45,8 @@ LOG = logging.getLogger(__name__) if TYPE_CHECKING: - from benchbuild.project import Project from benchbuild.experiment import Experiment + from benchbuild.project import Project def strip_path_prefix(ipath: str, prefix: str) -> str: @@ -300,7 +300,7 @@ def persist(id_obj, filename=None, suffix=None): ident = str(id(id_obj)) if filename is None: - filename = "{obj_id}{suffix}".format(obj_id=ident, suffix=suffix) + filename = f'{ident}{suffix}' with open(filename, 'wb') as obj_file: dill.dump(id_obj, obj_file) From 9d6b9dd54d8a41718a8b76e7cb108aeca90cd346 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 17 Jan 2022 01:38:15 +0100 Subject: [PATCH 15/32] fix(environments): use enumerate_projects instead of projects attribute --- benchbuild/environments/entrypoints/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchbuild/environments/entrypoints/cli.py b/benchbuild/environments/entrypoints/cli.py index a6db0f4e7..50ed82260 100644 --- a/benchbuild/environments/entrypoints/cli.py +++ b/benchbuild/environments/entrypoints/cli.py @@ -456,7 +456,7 @@ def create_experiment_images( """ publish = bootstrap.bus() for exp in enumerate_experiments(experiments, projects): - for prj in exp.projects: + for prj in enumerate_projects(experiments, projects): version = make_version_tag(*prj.variant.values()) base_tag = make_image_name(f'{prj.name}/{prj.group}', version) image_tag = make_image_name( @@ -490,7 +490,7 @@ def run_experiment_images( publish = bootstrap.bus() for exp in enumerate_experiments(experiments, projects): - for prj in exp.projects: + for prj in enumerate_projects(experiments, projects): version = make_version_tag(*prj.variant.values()) image_tag = make_image_name( f'{exp.name}/{prj.name}/{prj.group}', version From 1c6b405f6f116995c36c6b9383655f8315e09630 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 17 Jan 2022 02:07:38 +0100 Subject: [PATCH 16/32] fix(container): add some easy type annotations --- benchbuild/container.py | 53 ++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/benchbuild/container.py b/benchbuild/container.py index 1f87cac6a..f28baf36d 100644 --- a/benchbuild/container.py +++ b/benchbuild/container.py @@ -12,17 +12,20 @@ from abc import abstractmethod from plumbum import FG, TF, ProcessExecutionError, cli, local +from plumbum.path.local import LocalPath from benchbuild.settings import CFG from benchbuild.utils import bootstrap, container, download, log, run, uchroot from benchbuild.utils import user_interface as ui from benchbuild.utils.cmd import bash, mkdir, mv, rm, tar -from benchbuild.utils.settings import get_number_of_jobs +from benchbuild.utils.settings import get_number_of_jobs, Configuration LOG = logging.getLogger(__name__) -def clean_directories(builddir, in_dir=True, out_dir=True): +def clean_directories( + builddir: str, in_dir: bool = True, out_dir: bool = True +) -> None: """Remove the in and out of the container if confirmed by the user.""" container_in = local.path(builddir) / "container-in" container_out = local.path(builddir) / "container-out" @@ -35,7 +38,7 @@ def clean_directories(builddir, in_dir=True, out_dir=True): container_out.delete() -def setup_directories(builddir): +def setup_directories(builddir: str) -> None: """Create the in and out directories of the container.""" build_dir = local.path(builddir) in_dir = build_dir / "container-in" @@ -47,7 +50,7 @@ def setup_directories(builddir): out_dir.mkdir() -def setup_container(builddir, _container): +def setup_container(builddir: str, _container: str) -> LocalPath: """Prepare the container and returns the path where it can be found.""" build_dir = local.path(builddir) in_dir = build_dir / "container-in" @@ -83,7 +86,7 @@ def setup_container(builddir, _container): return in_dir -def run_in_container(command, container_dir): +def run_in_container(command, container_dir: str): """ Run a given command inside a container. @@ -108,7 +111,7 @@ def run_in_container(command, container_dir): return cmd & FG -def pack_container(in_container, out_file): +def pack_container(in_container: str, out_file: str) -> None: """ Pack a container image into a .tar.bz2 archive. @@ -133,7 +136,9 @@ def pack_container(in_container, out_file): CFG["container"]["known"] += new_container -def setup_bash_in_container(builddir, _container, outfile, shell): +def setup_bash_in_container( + builddir: str, _container: str, outfile: str, shell +) -> None: """ Setup a bash environment inside a container. @@ -162,7 +167,7 @@ def setup_bash_in_container(builddir, _container, outfile, shell): print(f'Storing config in {store_path}') -def find_hash(container_db, key): +def find_hash(container_db, key: str): """Find the first container in the database with the given key.""" for keyvalue in container_db: if keyvalue["hash"].startswith(key): @@ -170,7 +175,7 @@ def find_hash(container_db, key): return None -def set_input_container(_container, cfg): +def set_input_container(_container: LocalPath, cfg: Configuration) -> bool: """Save the input for the container in the configurations.""" if not _container: return False @@ -219,7 +224,7 @@ def run(self, context): class SetupPolyJITGentooStrategy(ContainerStrategy): """Interface of using gentoo as a container for an experiment.""" - def run(self, context): + def run(self, context) -> None: """Setup a gentoo container suitable for PolyJIT.""" # Don't do something when running non-interactive. if not sys.stdout.isatty(): @@ -274,23 +279,23 @@ def run(self, context): class Container(cli.Application): """Manage uchroot containers.""" - VERSION = str(CFG["version"]) + VERSION: str = str(CFG["version"]) @cli.switch(["-i", "--input-file"], str, help="Input container path") - def input_file(self, _container): + def input_file(self, _container: str) -> None: """Find the input path of a uchroot container.""" p = local.path(_container) if set_input_container(p, CFG): return - p = find_hash(CFG["container"]["known"].value, container) + p = find_hash(CFG["container"]["known"].value, _container) if set_input_container(p, CFG): return raise ValueError(f'The path \'{p}\' does not exist.') @cli.switch(["-o", "--output-file"], str, help="Output container path") - def output_file(self, _container): + def output_file(self, _container: str) -> None: """Find and writes the output path of a chroot container.""" p = local.path(_container) if p.exists(): @@ -302,14 +307,14 @@ def output_file(self, _container): @cli.switch(["-s", "--shell"], str, help="The shell command we invoke inside the container.") - def shell(self, custom_shell): + def shell(self, custom_shell: str) -> None: """The command to run inside the container.""" CFG["container"]["shell"] = custom_shell @cli.switch(["-t", "-tmp-dir"], cli.ExistingDirectory, help="Temporary directory") - def builddir(self, tmpdir): + def builddir(self, tmpdir: str) -> None: """Set the current builddir of the container.""" CFG["build_dir"] = tmpdir @@ -319,13 +324,13 @@ def builddir(self, tmpdir): list=True, help="Mount the given directory under / inside the uchroot container" ) - def mounts(self, user_mount): + def mounts(self, user_mount: str) -> None: """Save the current mount of the container into the settings.""" CFG["container"]["mounts"] = user_mount - verbosity = cli.CountOf('-v', help="Enable verbose output") + verbosity: int = cli.CountOf('-v', help="Enable verbose output") - def main(self, *args): + def main(self, *args) -> None: log.configure() builddir = local.path(str(CFG["build_dir"])) if not builddir.exists(): @@ -345,7 +350,7 @@ def main(self, *args): class ContainerRun(cli.Application): """Execute commannds inside a prebuilt container.""" - def main(self, *args): + def main(self, *args) -> None: builddir = str(CFG["build_dir"]) in_container = str(CFG["container"]["input"]) @@ -394,7 +399,7 @@ def strategy(self, strategy: str): "polyjit": SetupPolyJITGentooStrategy() }[strategy] - def main(self, *args): + def main(self, *args) -> None: builddir = str(CFG["build_dir"]) in_container = str(CFG["container"]["input"]) out_container = str(CFG["container"]["output"]) @@ -424,7 +429,7 @@ def main(self, *args): class ContainerBootstrap(cli.Application): """Check for the needed files.""" - def install_cmake_and_exit(self): + def install_cmake_and_exit(self) -> None: """Tell the user to install cmake and aborts the current process.""" print( "You need to install cmake via your package manager manually." @@ -432,7 +437,7 @@ def install_cmake_and_exit(self): ) sys.exit(-1) - def main(self, *args): + def main(self, *args) -> None: print("Checking container binary dependencies...") if not bootstrap.find_package("uchroot"): if not bootstrap.find_package("cmake"): @@ -455,7 +460,7 @@ def main(self, *args): class ContainerList(cli.Application): """Prints a list of the known containers.""" - def main(self, *args): + def main(self, *args) -> None: containers = CFG["container"]["known"].value for c in containers: c_path = c["path"] From 48bbb71af90165f2938cae376d03bd347b2cc00f Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 17 Jan 2022 02:08:08 +0100 Subject: [PATCH 17/32] fix(utils): add some more easy type annotations --- benchbuild/extensions/base.py | 4 ++-- benchbuild/utils/settings.py | 2 +- benchbuild/utils/wrapping.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/benchbuild/extensions/base.py b/benchbuild/extensions/base.py index 48bb5c6f5..46b7edb3d 100644 --- a/benchbuild/extensions/base.py +++ b/benchbuild/extensions/base.py @@ -38,7 +38,7 @@ class Extension(metaclass=ABCMeta): def __init__( self, *extensions: Extension, - config: tp.Optional[tp.Dict[str, str]] = None, + config: tp.Optional[tp.Dict[str, tp.Any]] = None, **kwargs: tp.Any ): """Initialize an extension with an arbitrary number of children.""" @@ -86,7 +86,7 @@ def print(self, indent: int = 0) -> None: ext.print(indent=indent + 2) def __call__(self, command: BoundCommand, *args: str, - **kwargs: tp.Any) -> tp.List[run.RunInfo]: + **kwargs: tp.Any) -> tp.Optional[tp.List[run.RunInfo]]: return self.call_next(*args, **kwargs) def __str__(self) -> str: diff --git a/benchbuild/utils/settings.py b/benchbuild/utils/settings.py index 12857547a..d3f2994b0 100644 --- a/benchbuild/utils/settings.py +++ b/benchbuild/utils/settings.py @@ -104,7 +104,7 @@ def current_available_threads() -> int: return len(os.sched_getaffinity(0)) -def get_number_of_jobs(config: Configuration) -> int: +def get_number_of_jobs(config: Configuration | tp.Dict[str, tp.Any]) -> int: """Returns the number of jobs set in the config.""" jobs_configured = int(config["jobs"]) if jobs_configured == 0: diff --git a/benchbuild/utils/wrapping.py b/benchbuild/utils/wrapping.py index 072e70037..2672684b8 100644 --- a/benchbuild/utils/wrapping.py +++ b/benchbuild/utils/wrapping.py @@ -131,14 +131,14 @@ def wrap( project_file = persist(project, suffix=".project") - env = CFG['env'].value + cfg_env = dict(CFG['env'].value) - bin_path = list_to_path(env.get('PATH', [])) + bin_path = list_to_path(cfg_env.get('PATH', [])) bin_path = list_to_path([bin_path, os.environ["PATH"]]) - bin_lib_path = list_to_path(env.get('LD_LIBRARY_PATH', [])) + bin_lib_path = list_to_path(cfg_env.get('LD_LIBRARY_PATH', [])) bin_lib_path = list_to_path([bin_lib_path, os.environ["LD_LIBRARY_PATH"]]) - home = env.get("HOME", os.getenv("HOME", "")) + home = cfg_env.get("HOME", os.getenv("HOME", "")) with open(name_absolute, 'w') as wrapper: wrapper.write( From 55ec3211317c88f9b7c7f3ece140a9049915b839 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 24 Jan 2022 01:42:49 +0100 Subject: [PATCH 18/32] refactor(typing): add a bunch of additional type annotations. --- benchbuild/likwid.py | 7 ++++- benchbuild/signals.py | 19 +++++++----- benchbuild/utils/dict.py | 48 +++++++++++++++++++----------- benchbuild/utils/settings.py | 42 ++++++++++++++++++-------- benchbuild/utils/user_interface.py | 9 +++--- tests/settings/test_yaml.py | 2 +- 6 files changed, 82 insertions(+), 45 deletions(-) diff --git a/benchbuild/likwid.py b/benchbuild/likwid.py index 071f41762..2aff59cce 100644 --- a/benchbuild/likwid.py +++ b/benchbuild/likwid.py @@ -128,7 +128,12 @@ def read_tables( table = read_table(fstream) -def get_measurements(region: str, core_info, data, extra_offset: int = 0): +def get_measurements( + region: str, + core_info, + data, + extra_offset: int = 0 +) -> tp.List[tp.Tuple[str, str, str, str]]: """ Get the complete measurement info from likwid's region info. diff --git a/benchbuild/signals.py b/benchbuild/signals.py index c8968b5db..8c44922c2 100644 --- a/benchbuild/signals.py +++ b/benchbuild/signals.py @@ -6,33 +6,36 @@ LOG = logging.getLogger(__name__) +StoredProcedureTy = tp.Callable[..., None] +StoredProceduresTy = tp.Dict[StoredProcedureTy, tp.Callable[[], None]] + class CleanupOnSignal: - __stored_procedures: tp.Dict[tp.Callable, tp.Callable] = {} + __stored_procedures: StoredProceduresTy = {} @property - def stored_procedures(self): + def stored_procedures(self) -> StoredProceduresTy: return self.__stored_procedures def register( - self, callback: tp.Callable, *args: tp.Any, **kwargs: tp.Any + self, callback: StoredProcedureTy, *args: tp.Any, **kwargs: tp.Any ) -> None: new_func = functools.partial(callback, *args, **kwargs) self.__stored_procedures[callback] = new_func - def deregister(self, callback): + def deregister(self, callback: StoredProcedureTy) -> None: del self.__stored_procedures[callback] - def __call__(self): - for k in self.stored_procedures: + def __call__(self) -> None: + for k, func in self.stored_procedures.items(): LOG.debug("Running stored cleanup procedure: %r", k) - self.stored_procedures[k]() + func() handlers = CleanupOnSignal() -def __handle_sigterm(signum, frame): +def __handle_sigterm(signum: int, frame: tp.Any) -> None: del frame LOG.debug("Got SIGTERM, running cleanup handlers") handlers() diff --git a/benchbuild/utils/dict.py b/benchbuild/utils/dict.py index 2baf5b59c..6970f8072 100644 --- a/benchbuild/utils/dict.py +++ b/benchbuild/utils/dict.py @@ -3,8 +3,9 @@ from contextlib import contextmanager -def extend_as_list(original_dict: tp.MutableMapping[tp.Any, tp.Any], - **kwargs) -> tp.Dict[tp.Any, tp.Any]: +def extend_as_list( + original_dict: tp.MutableMapping[tp.Any, tp.Any], **kwargs: tp.Any +) -> tp.Dict[tp.Any, tp.Any]: """ Extend values in a map by treating them as a list. """ @@ -27,18 +28,26 @@ def extend_as_list(original_dict: tp.MutableMapping[tp.Any, tp.Any], return dict(new_dict) +AnyDictTy = tp.Dict[tp.Any, tp.Any] +ExtenderFnTy = tp.Callable[[AnyDictTy], AnyDictTy] + + class ExtensibleDict: """A dictionary that provides temporary modification.""" - _current: tp.MutableMapping[tp.Any, tp.Any] = {} + _current: AnyDictTy = {} _extender_fn = None - def __init__(self, extender_fn=None): + def __init__(self, extender_fn: tp.Optional[ExtenderFnTy] = None): self._extender_fn = extender_fn super().__init__() @contextmanager - def __call__(self, *args, extender_fn=None, **kwargs): + def __call__( + self, + extender_fn: tp.Optional[ExtenderFnTy] = None, + **kwargs: tp.Any + ) -> tp.Generator[None, None, None]: """ A context manager to temporarily modify the content of dict. @@ -67,16 +76,16 @@ def __call__(self, *args, extender_fn=None, **kwargs): finally: self._current = previous - def __iter__(self): + def __iter__(self) -> tp.Iterator[tp.Tuple[tp.Any, tp.Any]]: return iter(self._current.items()) - def __len__(self): + def __len__(self) -> int: return len(self._current) - def __contains__(self, name): + def __contains__(self, name: tp.Any) -> bool: return name in self._current - def __getitem__(self, name): + def __getitem__(self, name: str) -> tp.Any: return self._current[name] def keys(self): @@ -88,32 +97,35 @@ def values(self): def items(self): return self._current.items() - def get(self, name, *default): + def get(self, name: tp.Any, *default: tp.Any) -> tp.Any: return self._current.get(name, *default) - def __delitem__(self, name): + def __delitem__(self, name: tp.Any) -> None: del self._current[name] - def __setitem__(self, name, value): + def __setitem__(self, name: tp.Any, value: tp.Any) -> None: self._current[name] = value - def pop(self, name, *default): + def pop(self, name: tp.Any, *default: tp.Any) -> tp.Any: return self._current.pop(name, *default) - def clear(self): + def clear(self) -> None: self._current.clear() - def update(self, extender_fn, *args, **kwargs): + def update( + self, extender_fn: tp.Optional[ExtenderFnTy], *args: tp.Any, + **kwargs: tp.Any + ) -> None: if extender_fn is not None: self._current.update(*args, **extender_fn(self._current, **kwargs)) else: self._current.update(*args, **kwargs) - def getdict(self): + def getdict(self) -> tp.Dict[tp.Any, tp.Any]: return dict((k, str(v)) for k, v in self._current.items()) - def __str__(self): + def __str__(self) -> str: return str(self._current) - def __repr__(self): + def __repr__(self) -> str: return repr(self._current) diff --git a/benchbuild/utils/settings.py b/benchbuild/utils/settings.py index d3f2994b0..39072ef51 100644 --- a/benchbuild/utils/settings.py +++ b/benchbuild/utils/settings.py @@ -12,6 +12,7 @@ from __future__ import annotations import copy +import io import logging import os import re @@ -154,8 +155,10 @@ class ConfigDumper(yaml.SafeDumper): def to_yaml(value: tp.Any) -> tp.Optional[str]: """Convert a given value to a YAML string.""" - stream = yaml.io.StringIO() - dumper = ConfigDumper(stream, default_flow_style=True, width=sys.maxsize) + stream = io.StringIO() + dumper = ConfigDumper( + tp.cast(tp.IO, stream), default_flow_style=True, width=sys.maxsize + ) val = None try: dumper.open() @@ -371,6 +374,9 @@ def __setitem__(self, key: str, val: tp.Any) -> None: else: self.node[key] = {'value': val} + def __iter__(self) -> tp.Iterator[str]: + return iter(self.node) + def __iadd__(self, rhs: tp.Any) -> tp.Any: """Append a value to a list value.""" if not self.has_value(): @@ -487,19 +493,23 @@ def __str__(self) -> str: return ConfigPath.path_to_str(self.components) -def path_representer(dumper, data): +def path_representer( + dumper: yaml.SafeDumper, data: ConfigPath +) -> yaml.ScalarNode: """ Represent a ConfigPath object as a scalar YAML node. """ - return dumper.represent_scalar('!create-if-needed', '%s' % data) + return dumper.represent_scalar('!create-if-needed', str(data)) -def path_constructor(loader, node): +def path_constructor(loader: yaml.SafeLoader, node: yaml.Node) -> ConfigPath: """" Construct a ConfigPath object form a scalar YAML node. """ + assert isinstance(node, yaml.ScalarNode) + value = loader.construct_scalar(node) - return ConfigPath(value) + return ConfigPath(str(value)) def find_config( @@ -630,24 +640,32 @@ def upgrade(cfg: Configuration) -> None: name=cfg["db"]["name"]["value"]) -def uuid_representer(dumper, data): +def uuid_representer( + dumper: yaml.SafeDumper, data: uuid.UUID +) -> yaml.ScalarNode: """Represent a uuid.UUID object as a scalar YAML node.""" - return dumper.represent_scalar('!uuid', '%s' % data) + return dumper.represent_scalar('!uuid', str(data)) -def uuid_constructor(loader, node): +def uuid_constructor(loader: yaml.SafeLoader, node: yaml.Node) -> uuid.UUID: """"Construct a uuid.UUID object form a scalar YAML node.""" + assert isinstance(node, yaml.ScalarNode) value = loader.construct_scalar(node) - return uuid.UUID(value) + return uuid.UUID(str(value)) -def uuid_add_implicit_resolver(loader=ConfigLoader, dumper=ConfigDumper): +def uuid_add_implicit_resolver( + loader_cls: tp.Type[ConfigLoader] = ConfigLoader, + dumper_cls: tp.Type[ConfigDumper] = ConfigDumper +) -> None: """Attach an implicit pattern resolver for UUID objects.""" uuid_regex = r'^\b[a-f0-9]{8}-\b[a-f0-9]{4}-\b[a-f0-9]{4}-\b[a-f0-9]{4}-\b[a-f0-9]{12}$' pattern = re.compile(uuid_regex) - yaml.add_implicit_resolver('!uuid', pattern, Loader=loader, Dumper=dumper) + yaml.add_implicit_resolver( + '!uuid', pattern, Loader=loader_cls, Dumper=dumper_cls + ) def __init_module__() -> None: diff --git a/benchbuild/utils/user_interface.py b/benchbuild/utils/user_interface.py index e153f3ba7..f98602893 100644 --- a/benchbuild/utils/user_interface.py +++ b/benchbuild/utils/user_interface.py @@ -4,12 +4,13 @@ import logging import os import sys +import typing as tp LOG = logging.getLogger(__name__) # Taken from the following recipe: http://code.activestate.com/recipes/577058/ -def query_yes_no(question, default="yes"): +def query_yes_no(question: str, default: str = "yes") -> bool: """ Ask a yes/no question via raw_input() and return their answer. @@ -23,9 +24,7 @@ def query_yes_no(question, default="yes"): True, if 'yes', False otherwise. """ valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} - if default is None: - prompt = " [y/n] " - elif default == "yes": + if default == "yes": prompt = " [Y/n] " elif default == "no": prompt = " [y/N] " @@ -65,7 +64,7 @@ def ask( """ response = default_answer - def should_ignore_tty(): + def should_ignore_tty() -> bool: """ Check, if we want to ignore an opened tty result. """ diff --git a/tests/settings/test_yaml.py b/tests/settings/test_yaml.py index 2c81129fe..5616e0717 100644 --- a/tests/settings/test_yaml.py +++ b/tests/settings/test_yaml.py @@ -39,7 +39,7 @@ def test_uuid_resolver(self): uuid_in = {'test': uuid.UUID(TEST_UUID)} yaml.add_representer(uuid.UUID, uuid_representer, Dumper=FakeDumper) - uuid_add_implicit_resolver(loader=FakeLoader, dumper=FakeDumper) + uuid_add_implicit_resolver(loader_cls=FakeLoader, dumper_cls=FakeDumper) self.assertEqual( yaml.dump(uuid_in, Dumper=FakeDumper), From 8c43bbc9144d75e544b91e30ad3cea787f5ce552 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 24 Jan 2022 02:11:25 +0100 Subject: [PATCH 19/32] refactor(utils/download): add more type annotations/clean code. This adds more type annotations and removes long deprecated and untested code: with_wget, with_git. --- benchbuild/utils/download.py | 144 +++-------------------------------- 1 file changed, 10 insertions(+), 134 deletions(-) diff --git a/benchbuild/utils/download.py b/benchbuild/utils/download.py index 9264a1586..94a510f28 100644 --- a/benchbuild/utils/download.py +++ b/benchbuild/utils/download.py @@ -13,7 +13,8 @@ import hashlib import logging import os -from typing import Callable, List, Optional, Type +import typing as tp +from typing import Type from plumbum import local @@ -22,7 +23,7 @@ LOG = logging.getLogger(__name__) -AnyC = Type[object] +AnyC = tp.Type[object] def get_hash_of_dirs(directory: str) -> str: @@ -99,7 +100,7 @@ def update_hash(src_file: local.path) -> str: return new_hash -def Copy(From, To): +def Copy(From: str, To: str) -> None: """ Small copy wrapper. @@ -111,7 +112,7 @@ def Copy(From, To): cp("-ar", "--reflink=auto", From, To) -def CopyNoFail(src, root=None): +def CopyNoFail(src: str, root: tp.Optional[str] = None) -> bool: """ Just copy fName into the current working directory, if it exists. @@ -135,7 +136,9 @@ def CopyNoFail(src, root=None): return False -def Wget(src_url, tgt_name, tgt_root=None): +def Wget( + src_url: str, tgt_name: str, tgt_root: tp.Optional[str] = None +) -> None: """ Download url, if required. @@ -160,44 +163,6 @@ def Wget(src_url, tgt_name, tgt_root=None): Copy(tgt_file, ".") -def with_wget(url_dict=None, target_file=None): - """ - Decorate a project class with wget-based version information. - - This adds two attributes to a project class: - - A `versions` method that returns a list of available versions - for this project. - - A `repository` attribute that provides a repository string to - download from later. - We use the `git rev-list` subcommand to list available versions. - - Args: - url_dict (dict): A dictionary that assigns a version to a download URL. - target_file (str): An optional path where we should put the clone. - If unspecified, we will use the `SRC_FILE` attribute of - the decorated class. - """ - - def wget_decorator(cls): - - def download_impl(self): - """Download the selected version from the url_dict value.""" - t_file = target_file if target_file else self.SRC_FILE - t_version = url_dict[self.version] - Wget(t_version, t_file) - - @staticmethod - def versions_impl(): - """Return a list of versions from the url_dict keys.""" - return list(url_dict.keys()) - - cls.versions = versions_impl - cls.download = download_impl - return cls - - return wget_decorator - - def __clone_needed__(repository: str, directory: str) -> bool: """ Do we need to create a fresh clone of the given repository. @@ -272,96 +237,7 @@ def Git( return repository_loc -def with_git( - repo: str, - target_dir: Optional[str] = None, - limit: Optional[int] = None, - refspec: str = "HEAD", - clone: bool = True, - rev_list_args: Optional[List[str]] = None, - shallow_clone: bool = True, - version_filter: Callable[[str], bool] = lambda version: True -) -> Callable[[AnyC], AnyC]: - """ - Decorate a project class with git-based version information. - - This adds two attributes to a project class: - - A `versions` method that returns a list of available versions - for this project. - - A `repository` attribute that provides a repository string to - download from later. - We use the `git rev-list` subcommand to list available versions. - - Args: - repo (str): Repository to download from, this will be stored - in the `repository` attribute of the decorated class. - target_dir (str): An optional path where we should put the clone. - If unspecified, we will use the `SRC_FILE` attribute of - the decorated class. - limit (int): Limit the number of commits to consider for available - versions. Versions are 'ordered' from latest to oldest. - refspec (str): A git refspec string to start listing the versions from. - clone (bool): Should we clone the repo if it isn't already available - in our tmp dir? Defaults to `True`. You can set this to False to - avoid time consuming clones, when the project has not been accessed - at least once in your installation. - ref_list_args (list of str): Additional arguments you want to pass to - `git rev-list`. - shallow_clone (bool): Only clone the repository shallow - Defaults to true - version_filter (class filter): Filter function to remove unwanted - project versions. - - """ - if not rev_list_args: - rev_list_args = [] - - def git_decorator(cls): - from benchbuild.utils.cmd import git - - @staticmethod - def versions_impl(): - """Return a list of versions from the git hashes up to :limit:.""" - directory = cls.SRC_FILE if target_dir is None else target_dir - repo_prefix = local.path(str(CFG["tmp_dir"])) - repo_loc = local.path(repo_prefix) / directory - if __clone_needed__(repo, repo_loc): - if not clone: - return [] - git("clone", repo, repo_loc) - - with local.cwd(repo_loc): - rev_list = git( - "rev-list", "--abbrev-commit", "--abbrev=10", refspec, - *rev_list_args - ).strip().split('\n') - git("rev-parse", "--short=10", refspec).strip().split('\n') - - if limit: - return list(filter(version_filter, rev_list))[:limit] - - return list(filter(version_filter, rev_list)) - - def download_impl(self): - """Download the selected version.""" - nonlocal target_dir, git - directory = cls.SRC_FILE if target_dir is None else target_dir - Git( - self.repository, - directory, - self.version, - shallow_clone=shallow_clone - ) - - cls.versions = versions_impl - cls.download = download_impl - cls.repository = repo - return cls - - return git_decorator - - -def Svn(url, fname, to=None): +def Svn(url: str, fname: str, to: tp.Optional[str] = None) -> None: """ Checkout the SVN repo. @@ -385,7 +261,7 @@ def Svn(url, fname, to=None): Copy(src_dir, ".") -def Rsync(url, tgt_name, tgt_root=None): +def Rsync(url: str, tgt_name: str, tgt_root: tp.Optional[str] = None) -> None: """ RSync a folder. From 8b40d79be757b842f8b2e2ea1f9202d88f8bc437 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 24 Jan 2022 02:13:00 +0100 Subject: [PATCH 20/32] refactor(typing): more annotations --- benchbuild/environments/domain/declarative.py | 2 +- benchbuild/utils/__init__.py | 15 +++++++-------- benchbuild/utils/path.py | 6 +++++- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/benchbuild/environments/domain/declarative.py b/benchbuild/environments/domain/declarative.py index 76a523ecf..d2b34face 100644 --- a/benchbuild/environments/domain/declarative.py +++ b/benchbuild/environments/domain/declarative.py @@ -23,7 +23,7 @@ LOG = logging.getLogger(__name__) -class ContainerImage(list): +class ContainerImage(list[model.Layer]): """ Define a container image declaratively. diff --git a/benchbuild/utils/__init__.py b/benchbuild/utils/__init__.py index 80188c091..2de141e78 100644 --- a/benchbuild/utils/__init__.py +++ b/benchbuild/utils/__init__.py @@ -25,12 +25,12 @@ class ErrorCommand(LocalCommand): """ EXE = __name__ + ".error_cmd" - def run(self, *args, **kwargs): + def run(self, *args: tp.Any, **kwargs: tp.Any) -> None: """Simply raises the AttributeError for a missing command.""" LOG.error("Unable to import a needed module.") raise AttributeError(self.EXE) - def popen(self, *args, **kwargs): + def popen(self, *args: tp.Any, **kwargs: tp.Any) -> None: """Simply raises the AttributeError for a missing command.""" LOG.error("Unable to import a needed module.") raise AttributeError(self.EXE) @@ -47,14 +47,13 @@ class CommandAlias(ModuleType): __all__ = () __package__ = __name__ - __overrides__ = {} - __override_all__ = None + __overrides__: tp.Dict[str, tp.List[str]] = {} + __override_all__: tp.Optional[pb.commands.ConcreteCommand] = None def __getattr__(self, command: str) -> pb.commands.ConcreteCommand: """Proxy getter for plumbum commands.""" from benchbuild.settings import CFG - from benchbuild.utils.path import list_to_path - from benchbuild.utils.path import path_to_list + from benchbuild.utils.path import list_to_path, path_to_list check = [command] @@ -89,10 +88,10 @@ def __getattr__(self, command: str) -> pb.commands.ConcreteCommand: LOG.debug("'%s' cannot be found. Import failed.", command) return ERROR - def __getitem__(self, command): + def __getitem__(self, command: str) -> pb.commands.ConcreteCommand: return self.__getattr__(command) - __path__ = [] + __path__: tp.List[str] = [] __file__ = __file__ diff --git a/benchbuild/utils/path.py b/benchbuild/utils/path.py index 069f82e62..035b01406 100644 --- a/benchbuild/utils/path.py +++ b/benchbuild/utils/path.py @@ -1,6 +1,7 @@ """ Path utilities for benchbuild. """ import fcntl import os +import typing as tp from contextlib import contextmanager from typing import List, Optional @@ -129,7 +130,10 @@ def mkdir_interactive(dirpath: str) -> None: @contextmanager -def flocked(filename: str, lock_type: int = fcntl.LOCK_EX): +def flocked( + filename: str, + lock_type: int = fcntl.LOCK_EX +) -> tp.Generator[tp.TextIO, None, None]: """ Lock a section using fcntl. From f18c3c8816aaf1a263f5c0b73cf199ad4fb4a8c7 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Mon, 24 Jan 2022 02:18:11 +0100 Subject: [PATCH 21/32] fix(declarative): unbreak CI --- benchbuild/environments/domain/declarative.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchbuild/environments/domain/declarative.py b/benchbuild/environments/domain/declarative.py index d2b34face..76a523ecf 100644 --- a/benchbuild/environments/domain/declarative.py +++ b/benchbuild/environments/domain/declarative.py @@ -23,7 +23,7 @@ LOG = logging.getLogger(__name__) -class ContainerImage(list[model.Layer]): +class ContainerImage(list): """ Define a container image declaratively. From 5e95b76f6b327a13ab66c7f0f6279c717f3011ea Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Tue, 25 Jan 2022 23:01:03 +0100 Subject: [PATCH 22/32] refactor(utils/dict): type annotations --- benchbuild/utils/dict.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchbuild/utils/dict.py b/benchbuild/utils/dict.py index 6970f8072..eb9ecf7a4 100644 --- a/benchbuild/utils/dict.py +++ b/benchbuild/utils/dict.py @@ -88,13 +88,13 @@ def __contains__(self, name: tp.Any) -> bool: def __getitem__(self, name: str) -> tp.Any: return self._current[name] - def keys(self): + def keys(self) -> tp.KeysView[tp.Any]: return self._current.keys() - def values(self): + def values(self) -> tp.ValuesView[tp.Any]: return self._current.values() - def items(self): + def items(self) -> tp.ItemsView[tp.Any, tp.Any]: return self._current.items() def get(self, name: tp.Any, *default: tp.Any) -> tp.Any: From 9a08544d6c77dfcd83b812158e040861e6f796d1 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Tue, 25 Jan 2022 23:01:28 +0100 Subject: [PATCH 23/32] refactor(utils/wrapping): type annotations --- benchbuild/utils/wrapping.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/benchbuild/utils/wrapping.py b/benchbuild/utils/wrapping.py index 2672684b8..e8d6b1bf5 100644 --- a/benchbuild/utils/wrapping.py +++ b/benchbuild/utils/wrapping.py @@ -26,6 +26,7 @@ import os import sys import typing as tp +import uuid from typing import TYPE_CHECKING import dill @@ -277,7 +278,16 @@ def wrap_cc( return local[filepath] -def persist(id_obj, filename=None, suffix=None): +@tp.runtime_checkable +class Identifiable(tp.Protocol): + run_uuid: uuid.UUID + + +def persist( + id_obj: Identifiable, + filename: tp.Optional[str] = None, + suffix: tp.Optional[str] = None +) -> str: """Persist an object in the filesystem. This will generate a pickled version of the given obj in the filename path. @@ -295,7 +305,7 @@ def persist(id_obj, filename=None, suffix=None): if suffix is None: suffix = ".pickle" if hasattr(id_obj, 'run_uuid'): - ident = id_obj.run_uuid + ident = str(id_obj.run_uuid) else: ident = str(id(id_obj)) From 1a9b582d45451de152ed0417d4fb7fae1dca7925 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Tue, 25 Jan 2022 23:04:38 +0100 Subject: [PATCH 24/32] refactor(extensions/base): type annotations --- benchbuild/extensions/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchbuild/extensions/base.py b/benchbuild/extensions/base.py index 46b7edb3d..3f89b6297 100644 --- a/benchbuild/extensions/base.py +++ b/benchbuild/extensions/base.py @@ -108,5 +108,5 @@ class MissingExtension(Extension): existing old experiments. """ - def __call__(self, *args, **kwargs) -> tp.List[run.RunInfo]: + def __call__(self, *args: tp.Any, **kwargs: tp.Any) -> tp.List[run.RunInfo]: raise ExtensionRequired() From c1c850ff07d1818687587c1f1510ecdcc0976b46 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 26 Jan 2022 00:33:16 +0100 Subject: [PATCH 25/32] refactor(database): type annotations (incomplete) --- benchbuild/utils/db.py | 79 ++++++++++++++++++++++-------- benchbuild/utils/run.py | 99 +++++++++++++++++++++++++------------- benchbuild/utils/schema.py | 24 +++++++-- 3 files changed, 146 insertions(+), 56 deletions(-) diff --git a/benchbuild/utils/db.py b/benchbuild/utils/db.py index 2302b9402..85cc0ef83 100644 --- a/benchbuild/utils/db.py +++ b/benchbuild/utils/db.py @@ -1,16 +1,29 @@ """Database support module for the benchbuild study.""" +from __future__ import annotations + import logging +import typing as tp from sqlalchemy.exc import IntegrityError from benchbuild.settings import CFG +if tp.TYPE_CHECKING: + import uuid + + from plumbum.commands import BaseCommand + + from benchbuild.utils import schema + LOG = logging.getLogger(__name__) -def validate(func): +def validate(func: tp.Callable[..., tp.Any]) -> tp.Callable[..., tp.Any]: - def validate_run_func(run, session, *args, **kwargs): + def validate_run_func( + run: 'schema.Run', session: 'schema.CanCommit', *args: tp.Any, + **kwargs: tp.Any + ) -> tp.Any: if run.status == 'failed': LOG.debug("Run failed. Execution of '%s' cancelled", str(func)) return None @@ -20,7 +33,8 @@ def validate_run_func(run, session, *args, **kwargs): return validate_run_func -def create_run(cmd, project, exp, grp): +def create_run(cmd: 'BaseCommand', project, exp, + grp: uuid.UUID) -> tp.Tuple['schema.Run', 'schema.CanCommit']: """ Create a new 'run' in the database. @@ -42,19 +56,23 @@ def create_run(cmd, project, exp, grp): from benchbuild.utils import schema as s session = s.Session() - run = s.Run(command=str(cmd), - project_name=project.name, - project_group=project.group, - experiment_name=exp.name, - run_group=str(grp), - experiment_group=exp.id) + run = s.Run( + command=str(cmd), + project_name=project.name, + project_group=project.group, + experiment_name=exp.name, + run_group=str(grp), + experiment_group=exp.id + ) session.add(run) session.commit() return (run, session) -def create_run_group(prj, experiment): +def create_run_group( + prj, experiment +) -> tp.Tuple['schema.RunGroup', 'schema.CanCommit']: """ Create a new 'run_group' in the database. @@ -166,7 +184,10 @@ def persist_experiment(experiment): @validate -def persist_time(run, session, timings): +def persist_time( + run: schema.Run, session: schema.CanCommit, + timings: tp.Sequence[tp.Tuple[float, float, float]] +) -> None: """ Persist the run results in the database. @@ -178,15 +199,20 @@ def persist_time(run, session, timings): from benchbuild.utils import schema as s for timing in timings: - session.add(s.Metric(name="time.user_s", value=timing[0], - run_id=run.id)) session.add( - s.Metric(name="time.system_s", value=timing[1], run_id=run.id)) - session.add(s.Metric(name="time.real_s", value=timing[2], - run_id=run.id)) + s.Metric(name="time.user_s", value=timing[0], run_id=run.id) + ) + session.add( + s.Metric(name="time.system_s", value=timing[1], run_id=run.id) + ) + session.add( + s.Metric(name="time.real_s", value=timing[2], run_id=run.id) + ) -def persist_perf(run, session, svg_path): +def persist_perf( + run: schema.Run, session: schema.CanCommit, svg_path: str +) -> None: """ Persist the flamegraph in the database. @@ -203,10 +229,18 @@ def persist_perf(run, session, svg_path): with open(svg_path, 'r') as svg_file: svg_data = svg_file.read() session.add( - s.Metadata(name="perf.flamegraph", value=svg_data, run_id=run.id)) + s.Metadata(name="perf.flamegraph", value=svg_data, run_id=run.id) + ) + +@tp.runtime_checkable +class HasRunId(tp.Protocol): + run_id: int -def persist_compilestats(run, session, stats): + +def persist_compilestats( + run: schema.Run, session: schema.CanCommit, stats: tp.Sequence[HasRunId] +) -> None: """ Persist the run results in the database. @@ -216,11 +250,16 @@ def persist_compilestats(run, session, stats): stats: The stats we want to store in the database. """ for stat in stats: + # FIXME: Why does sqlalchemy assume Optional[int] on a pk attribute? + assert run.id is not None + stat.run_id = run.id session.add(stat) -def persist_config(run, session, cfg): +def persist_config( + run: schema.Run, session: schema.CanCommit, cfg: tp.Mapping[str, str] +) -> None: """ Persist the configuration in as key-value pairs. diff --git a/benchbuild/utils/run.py b/benchbuild/utils/run.py index a14f8f570..fa7865c32 100644 --- a/benchbuild/utils/run.py +++ b/benchbuild/utils/run.py @@ -1,9 +1,11 @@ """Experiment helpers.""" +from __future__ import annotations + import datetime import functools import logging import sys -import typing as t +import typing as tp from contextlib import contextmanager import attr @@ -18,12 +20,17 @@ else: from typing import Protocol -CommandResult = t.Tuple[int, str, str] +if tp.TYPE_CHECKING: + import uuid + + from benchbuild.utils import db, schema + +CommandResult = tp.Tuple[int, str, str] class WatchableCommand(Protocol): - def __call__(self, *args: t.Any, **kwargs: t.Any) -> CommandResult: + def __call__(self, *args: tp.Any, **kwargs: tp.Any) -> CommandResult: ... @@ -52,7 +59,9 @@ class RunInfo: session (): """ - def __begin(self, command: BaseCommand, project, experiment, group): + def __begin( + self, command: BaseCommand, project, experiment, group: uuid.UUID + ) -> None: """ Begin a run in the database log. @@ -82,7 +91,7 @@ def __begin(self, command: BaseCommand, project, experiment, group): self.db_run = db_run self.session = session - def __end(self, stdout, stderr): + def __end(self, stdout: str, stderr: str) -> None: """ End a run in the database log (Successfully). @@ -95,6 +104,9 @@ def __end(self, stdout, stderr): stdout: The stdout we captured of the run. stderr: The stderr we capture of the run. """ + # FIXME: This needs a separate PR. Remove attrs from this module. + assert self.session is not None + from benchbuild.utils.schema import RunLog run_id = self.db_run.id @@ -107,10 +119,11 @@ def __end(self, stdout, stderr): self.db_run.end = datetime.datetime.now() self.db_run.status = 'completed' + self.session.add(log) self.session.add(self.db_run) - def __fail(self, retcode, stdout, stderr): + def __fail(self, retcode: int, stdout: str, stderr: str) -> None: """ End a run in the database log (Unsuccessfully). @@ -124,6 +137,10 @@ def __fail(self, retcode, stdout, stderr): stdout: The stdout we captured of the run. stderr: The stderr we capture of the run. """ + + # FIXME: This needs a separate PR. Remove attrs from this module. + assert self.session is not None + from benchbuild.utils.schema import RunLog run_id = self.db_run.id @@ -139,19 +156,21 @@ def __fail(self, retcode, stdout, stderr): self.session.add(log) self.session.add(self.db_run) - cmd = attr.ib(default=None, repr=False) - failed = attr.ib(default=False) + cmd: tp.Optional[BaseCommand] = attr.ib(default=None, repr=False) + failed: bool = attr.ib(default=False) project = attr.ib(default=None, repr=False) experiment = attr.ib(default=None, repr=False) - retcode = attr.ib(default=0) - stdout = attr.ib(default=attr.Factory(list), repr=False) - stderr = attr.ib(default=attr.Factory(list), repr=False) + retcode: int = attr.ib(default=0) + stdout: str = attr.ib(default=attr.Factory(str), repr=False) + stderr: str = attr.ib(default=attr.Factory(str), repr=False) db_run = attr.ib(init=False, default=None) - session = attr.ib(init=False, default=None, repr=False) - payload = attr.ib(init=False, default=None, repr=False) + session: tp.Optional[schema.CanCommit + ] = attr.ib(init=False, default=None, repr=False) + payload: tp.Optional[tp.Dict[str, tp.Any] + ] = attr.ib(init=False, default=None, repr=False) - def __attrs_post_init__(self): + def __attrs_post_init__(self) -> None: self.__begin( self.cmd, self.project, self.experiment, self.project.run_uuid ) @@ -160,20 +179,18 @@ def __attrs_post_init__(self): run_id = self.db_run.id settings.CFG["db"]["run_id"] = run_id - def add_payload(self, name, payload): - if self == payload: - return + def add_payload(self, name: str, payload: tp.Dict[str, tp.Any]) -> None: if not self.payload: self.payload = {name: payload} else: self.payload.update({name: payload}) @property - def has_failed(self): + def has_failed(self) -> bool: """Check, whether this run failed.""" return self.failed - def __call__(self, *args, expected_retcode=0, ri=None, **kwargs): + def __call__(self, expected_retcode: int = 0) -> RunInfo: cmd_env = settings.CFG.to_env_dict() with local.env(**cmd_env): @@ -213,11 +230,15 @@ def __call__(self, *args, expected_retcode=0, ri=None, **kwargs): return self - def commit(self): + def commit(self) -> None: + # FIXME: This needs a separate PR. Remove attrs from this module. + assert self.session is not None + self.session.commit() -def begin_run_group(project, experiment): +def begin_run_group(project, + experiment) -> tp.Tuple[schema.RunGroup, schema.CanCommit]: """ Begin a run_group in the database. @@ -241,7 +262,7 @@ def begin_run_group(project, experiment): return group, session -def end_run_group(group, session): +def end_run_group(group: schema.RunGroup, session: schema.CanCommit) -> None: """ End the run_group successfully. @@ -254,7 +275,7 @@ def end_run_group(group, session): session.commit() -def fail_run_group(group, session): +def fail_run_group(group: schema.RunGroup, session: schema.CanCommit) -> None: """ End the run_group unsuccessfully. @@ -268,7 +289,7 @@ def fail_run_group(group, session): def exit_code_from_run_infos( - run_infos: t.Union[RunInfo, t.List[RunInfo]] + run_infos: tp.Union[RunInfo, tp.List[RunInfo]] ) -> int: """Generate a single exit code from a list of RunInfo objects. @@ -276,7 +297,7 @@ def exit_code_from_run_infos( from 0. Args: - run_infos (t.List[RunInfo]): [description] + run_infos (tp.List[RunInfo]): [description] Returns: int: [description] @@ -295,7 +316,8 @@ def exit_code_from_run_infos( @contextmanager -def track_execution(cmd, project, experiment, **kwargs): +def track_execution(cmd: BaseCommand, project, experiment, + **kwargs: tp.Any) -> tp.Generator[RunInfo, None, None]: """Guard the execution of the given command. The given command (`cmd`) will be executed inside a database context. @@ -325,10 +347,10 @@ def watch(command: BaseCommand) -> WatchableCommand: command: The plumbumb command to execute. """ - def f(*args: t.Any, retcode: int = 0, **kwargs: t.Any) -> CommandResult: + def f(*args: tp.Any, retcode: int = 0, **kwargs: tp.Any) -> CommandResult: final_command = command[args] buffered = not bool(CFG['force_watch_unbuffered']) - return t.cast( + return tp.cast( CommandResult, final_command.run_tee(retcode=retcode, buffered=buffered, **kwargs) ) @@ -356,7 +378,12 @@ def with_env_recursive(cmd: BaseCommand, **envvars: str) -> BaseCommand: return cmd -def in_builddir(sub: str = '.'): +@tp.runtime_checkable +class HasBuilddir(tp.Protocol): + builddir: local.path + + +def in_builddir(sub: str = '.') -> tp.Callable[..., tp.Any]: """ Decorate a project phase with a local working directory change. @@ -364,11 +391,15 @@ def in_builddir(sub: str = '.'): sub: An optional subdirectory to change into. """ - def wrap_in_builddir(func): + def wrap_in_builddir( + func: tp.Callable[..., tp.Any] + ) -> tp.Callable[..., tp.Any]: """Wrap the function for the new build directory.""" @functools.wraps(func) - def wrap_in_builddir_func(self, *args, **kwargs): + def wrap_in_builddir_func( + self: HasBuilddir, *args: tp.Any, **kwargs: tp.Any + ) -> tp.Any: """The actual function inside the wrapper for the new builddir.""" p = local.path(self.builddir) / sub if not p.exists(): @@ -385,11 +416,13 @@ def wrap_in_builddir_func(self, *args, **kwargs): return wrap_in_builddir -def store_config(func): +def store_config(func: tp.Callable[..., tp.Any]) -> tp.Callable[..., tp.Any]: """Decorator for storing the configuration in the project's builddir.""" @functools.wraps(func) - def wrap_store_config(self, *args, **kwargs): + def wrap_store_config( + self: HasBuilddir, *args: tp.Any, **kwargs: tp.Any + ) -> tp.Any: """Wrapper that contains the actual storage call for the config.""" CFG.store(local.path(self.builddir) / ".benchbuild.yml") return func(self, *args, **kwargs) diff --git a/benchbuild/utils/schema.py b/benchbuild/utils/schema.py index 63684e075..e0a90ef72 100644 --- a/benchbuild/utils/schema.py +++ b/benchbuild/utils/schema.py @@ -20,6 +20,7 @@ As soon as we have alembic running, we can provide automatic up/downgrade paths for you. """ +from __future__ import annotations import functools import logging @@ -52,6 +53,23 @@ LOG = logging.getLogger(__name__) +# Type extensions for sqlalchemy are not hooked up properly yet. +@tp.runtime_checkable +class CanCommit(tp.Protocol): + + def commit(self) -> None: + ... + + def rollback(self) -> None: + ... + + def add(self, val: BASE) -> None: + ... + + def get(self) -> tp.Callable[[], CanCommit]: + ... + + def metadata(): return BASE.metadata @@ -508,12 +526,12 @@ def __del__(self): self.__transaction.rollback() -def __lazy_session__(): +def __lazy_session__() -> tp.Callable[[], CanCommit]: """Initialize the connection manager lazily.""" - connection_manager = None + connection_manager: tp.Optional[CanCommit] = None session = None - def __lazy_session_wrapped(): + def __lazy_session_wrapped() -> CanCommit: nonlocal connection_manager nonlocal session if connection_manager is None: From a64c14d6e79a67272a0881518a806b49cdc21c59 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 26 Jan 2022 00:38:29 +0100 Subject: [PATCH 26/32] fix(python): remove runtime_checkable (python 3.8+) --- benchbuild/utils/db.py | 1 - benchbuild/utils/run.py | 1 - benchbuild/utils/schema.py | 1 - benchbuild/utils/wrapping.py | 1 - 4 files changed, 4 deletions(-) diff --git a/benchbuild/utils/db.py b/benchbuild/utils/db.py index 85cc0ef83..7472b0c02 100644 --- a/benchbuild/utils/db.py +++ b/benchbuild/utils/db.py @@ -233,7 +233,6 @@ def persist_perf( ) -@tp.runtime_checkable class HasRunId(tp.Protocol): run_id: int diff --git a/benchbuild/utils/run.py b/benchbuild/utils/run.py index fa7865c32..7c95d12ef 100644 --- a/benchbuild/utils/run.py +++ b/benchbuild/utils/run.py @@ -378,7 +378,6 @@ def with_env_recursive(cmd: BaseCommand, **envvars: str) -> BaseCommand: return cmd -@tp.runtime_checkable class HasBuilddir(tp.Protocol): builddir: local.path diff --git a/benchbuild/utils/schema.py b/benchbuild/utils/schema.py index e0a90ef72..255508e37 100644 --- a/benchbuild/utils/schema.py +++ b/benchbuild/utils/schema.py @@ -54,7 +54,6 @@ # Type extensions for sqlalchemy are not hooked up properly yet. -@tp.runtime_checkable class CanCommit(tp.Protocol): def commit(self) -> None: diff --git a/benchbuild/utils/wrapping.py b/benchbuild/utils/wrapping.py index e8d6b1bf5..cb8f53006 100644 --- a/benchbuild/utils/wrapping.py +++ b/benchbuild/utils/wrapping.py @@ -278,7 +278,6 @@ def wrap_cc( return local[filepath] -@tp.runtime_checkable class Identifiable(tp.Protocol): run_uuid: uuid.UUID From d3d479d5941647ba2a073a419b4311da77be55dd Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 26 Jan 2022 00:44:13 +0100 Subject: [PATCH 27/32] fix(python): use typing_extensions (python 3.7) --- benchbuild/utils/db.py | 7 ++++++- benchbuild/utils/schema.py | 7 ++++++- benchbuild/utils/wrapping.py | 13 +++++++++---- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/benchbuild/utils/db.py b/benchbuild/utils/db.py index 7472b0c02..39fb505bb 100644 --- a/benchbuild/utils/db.py +++ b/benchbuild/utils/db.py @@ -8,6 +8,11 @@ from benchbuild.settings import CFG +if sys.version_info <= (3, 8): + from typing_extensions import Protocol +else: + from typing import Protocol + if tp.TYPE_CHECKING: import uuid @@ -233,7 +238,7 @@ def persist_perf( ) -class HasRunId(tp.Protocol): +class HasRunId(Protocol): run_id: int diff --git a/benchbuild/utils/schema.py b/benchbuild/utils/schema.py index 255508e37..8b1263d77 100644 --- a/benchbuild/utils/schema.py +++ b/benchbuild/utils/schema.py @@ -49,12 +49,17 @@ from benchbuild.utils import path from benchbuild.utils import user_interface as ui +if sys.version_info <= (3, 8): + from typing_extensions import Protocol +else: + from typing import Protocol + BASE = declarative_base() LOG = logging.getLogger(__name__) # Type extensions for sqlalchemy are not hooked up properly yet. -class CanCommit(tp.Protocol): +class CanCommit(Protocol): def commit(self) -> None: ... diff --git a/benchbuild/utils/wrapping.py b/benchbuild/utils/wrapping.py index cb8f53006..dbc9488ac 100644 --- a/benchbuild/utils/wrapping.py +++ b/benchbuild/utils/wrapping.py @@ -41,14 +41,19 @@ from benchbuild.utils.path import list_to_path from benchbuild.utils.uchroot import no_llvm as uchroot -PROJECT_BIN_F_EXT = ".bin" -PROJECT_BLOB_F_EXT = ".postproc" -LOG = logging.getLogger(__name__) +if sys.version_info <= (3, 8): + from typing_extensions import Protocol +else: + from typing import Protocol if TYPE_CHECKING: from benchbuild.experiment import Experiment from benchbuild.project import Project +PROJECT_BIN_F_EXT = ".bin" +PROJECT_BLOB_F_EXT = ".postproc" +LOG = logging.getLogger(__name__) + def strip_path_prefix(ipath: str, prefix: str) -> str: """ @@ -278,7 +283,7 @@ def wrap_cc( return local[filepath] -class Identifiable(tp.Protocol): +class Identifiable(Protocol): run_uuid: uuid.UUID From 5b74e10f9a86d0c74937d53ca9e11ccd7a0d024d Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 26 Jan 2022 01:00:38 +0100 Subject: [PATCH 28/32] refactor(extensions): type annotations --- benchbuild/extensions/base.py | 2 +- benchbuild/extensions/run.py | 36 ++++++++++++++++++++++------------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/benchbuild/extensions/base.py b/benchbuild/extensions/base.py index 3f89b6297..6c0ceffa0 100644 --- a/benchbuild/extensions/base.py +++ b/benchbuild/extensions/base.py @@ -86,7 +86,7 @@ def print(self, indent: int = 0) -> None: ext.print(indent=indent + 2) def __call__(self, command: BoundCommand, *args: str, - **kwargs: tp.Any) -> tp.Optional[tp.List[run.RunInfo]]: + **kwargs: tp.Any) -> tp.List[run.RunInfo]: return self.call_next(*args, **kwargs) def __str__(self) -> str: diff --git a/benchbuild/extensions/run.py b/benchbuild/extensions/run.py index ff8ddd963..c8a61710a 100644 --- a/benchbuild/extensions/run.py +++ b/benchbuild/extensions/run.py @@ -1,8 +1,10 @@ import logging import os +import typing as tp import yaml from plumbum import local +from plumbum.commands.base import BoundCommand from benchbuild.extensions import base from benchbuild.utils import db, run @@ -19,16 +21,19 @@ class RuntimeExtension(base.Extension): tracked execution of a wrapped binary. """ - def __init__(self, project, experiment, *extensions, config=None): + def __init__( + self, project, experiment, *extensions: base.Extension, config=None + ): self.project = project self.experiment = experiment super().__init__(*extensions, config=config) - def __call__(self, binary_command, *args, **kwargs): + def __call__(self, command: BoundCommand, *args: str, + **kwargs: tp.Any) -> tp.List[run.RunInfo]: self.project.name = kwargs.get("project_name", self.project.name) - cmd = binary_command[args] + cmd = command[args] with run.track_execution( cmd, self.project, self.experiment, **kwargs ) as _run: @@ -48,11 +53,11 @@ def __call__(self, binary_command, *args, **kwargs): db.persist_config( run_info.db_run, run_info.session, self.config ) - res = self.call_next(binary_command, *args, **kwargs) + res = self.call_next(command, *args, **kwargs) res.append(run_info) return res - def __str__(self): + def __str__(self) -> str: return "Run wrapped binary" @@ -64,15 +69,19 @@ class WithTimeout(base.Extension): the limit to a given value on extension construction. """ - def __init__(self, *extensions, limit="10m", **kwargs): + def __init__( + self, + *extensions: base.Extension, + limit: str = "10m", + **kwargs: tp.Any + ): super().__init__(*extensions, **kwargs) self.limit = limit - def __call__(self, binary_command, *args, **kwargs): + def __call__(self, command: BoundCommand, *args: str, + **kwargs: tp.Any) -> tp.List[run.RunInfo]: from benchbuild.utils.cmd import timeout - return self.call_next( - timeout[self.limit, binary_command], *args, **kwargs - ) + return self.call_next(timeout[self.limit, command], *args, **kwargs) class SetThreadLimit(base.Extension): @@ -82,7 +91,8 @@ class SetThreadLimit(base.Extension): variable OMP_NUM_THREADS. """ - def __call__(self, binary_command, *args, **kwargs): + def __call__(self, command: BoundCommand, *args: str, + **kwargs: tp.Any) -> tp.List[run.RunInfo]: from benchbuild.settings import CFG config = self.config @@ -94,10 +104,10 @@ def __call__(self, binary_command, *args, **kwargs): ret = None with local.env(OMP_NUM_THREADS=str(jobs)): - ret = self.call_next(binary_command, *args, **kwargs) + ret = self.call_next(command, *args, **kwargs) return ret - def __str__(self): + def __str__(self) -> str: return "Limit number of OpenMP threads" From 83478668e7ee47948b11c6a168b1923876c0c244 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 26 Jan 2022 01:01:44 +0100 Subject: [PATCH 29/32] fix(utils/db): add forgotten sys import --- benchbuild/utils/db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/benchbuild/utils/db.py b/benchbuild/utils/db.py index 39fb505bb..39980ad65 100644 --- a/benchbuild/utils/db.py +++ b/benchbuild/utils/db.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging +import sys import typing as tp from sqlalchemy.exc import IntegrityError From 5d2a077ade6d6414ad33c32bf9bca071165f6573 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 26 Jan 2022 01:12:05 +0100 Subject: [PATCH 30/32] refactor(extensions): type annotations --- benchbuild/extensions/compiler.py | 4 ++-- benchbuild/extensions/log.py | 14 +++++++------- benchbuild/extensions/time.py | 28 ++++++++++++++++++++++++---- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/benchbuild/extensions/compiler.py b/benchbuild/extensions/compiler.py index ad8d7aadc..ae7636cc0 100644 --- a/benchbuild/extensions/compiler.py +++ b/benchbuild/extensions/compiler.py @@ -9,8 +9,8 @@ from benchbuild.utils import db, run if TYPE_CHECKING: - from benchbuild.project import Project from benchbuild.experiment import Experiment + from benchbuild.project import Project LOG = logging.getLogger(__name__) @@ -80,5 +80,5 @@ def __call__( res.append(run_info) return res - def __str__(self): + def __str__(self) -> str: return "Compile /w fallback" diff --git a/benchbuild/extensions/log.py b/benchbuild/extensions/log.py index 4e143ab31..d120f5a36 100644 --- a/benchbuild/extensions/log.py +++ b/benchbuild/extensions/log.py @@ -10,9 +10,9 @@ class LogTrackingMixin: """Add log-registering capabilities to extensions.""" - _logs: tp.MutableSequence[str] = [] + _logs: tp.List[str] = [] - def add_log(self, path: str): + def add_log(self, path: str) -> None: """ Add a log to the tracked list. @@ -22,7 +22,7 @@ def add_log(self, path: str): self._logs.append(path) @property - def logs(self): + def logs(self) -> tp.List[str]: """Return list of tracked logs.""" return self._logs @@ -30,15 +30,15 @@ def logs(self): class LogAdditionals(base.Extension): """Log any additional log files that were registered.""" - def __call__(self, *args, **kwargs): + def __call__(self, *args: tp.Any, **kwargs: tp.Any) -> tp.List[run.RunInfo]: if not self.next_extensions: - return None + return [] res = self.call_next(*args, **kwargs) _cat = run.watch(cat) for ext in self.next_extensions: - if issubclass(ext.__class__, (LogTrackingMixin)): + if isinstance(ext, LogTrackingMixin): for log in ext.logs: LOG.debug("Dumping content of '%s'.", log) _cat(log) @@ -46,5 +46,5 @@ def __call__(self, *args, **kwargs): return res - def __str__(self): + def __str__(self) -> str: return "Dump additional log files" diff --git a/benchbuild/extensions/time.py b/benchbuild/extensions/time.py index 8976adbdd..cd8feed89 100644 --- a/benchbuild/extensions/time.py +++ b/benchbuild/extensions/time.py @@ -1,24 +1,44 @@ +from __future__ import annotations + import logging +import sys import typing as tp import parse +from plumbum.commands.base import BoundCommand from benchbuild.extensions import base from benchbuild.utils import db from benchbuild.utils.cmd import time +if sys.version_info <= (3, 8): + from typing_extensions import Protocol +else: + from typing import Protocol + +if tp.TYPE_CHECKING: + from benchbuild.utils import run + LOG = logging.getLogger(__name__) class RunWithTime(base.Extension): """Wrap a command with time and store the timings in the database.""" - def __call__(self, binary_command, *args, may_wrap=True, **kwargs): + def __call__( + self, + command: BoundCommand, + *args: str, + may_wrap: bool = True, + **kwargs: tp.Any + ) -> tp.List['run.RunInfo']: time_tag = "BENCHBUILD: " if may_wrap: - run_cmd = time["-f", time_tag + "%U-%S-%e", binary_command] + run_cmd = time["-f", time_tag + "%U-%S-%e", command] - def handle_timing(run_infos): + def handle_timing( + run_infos: tp.List['run.RunInfo'] + ) -> tp.List['run.RunInfo']: """Takes care of the formating for the timing statistics.""" from benchbuild.utils import schema as s @@ -39,7 +59,7 @@ def handle_timing(run_infos): res = self.call_next(run_cmd, *args, **kwargs) return handle_timing(res) - def __str__(self): + def __str__(self) -> str: return "Time execution of wrapped binary" From 043a975d05f59fdb51f0a2f8d24e39785076d492 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 26 Jan 2022 01:47:53 +0100 Subject: [PATCH 31/32] refactor(schema): add type annotations --- benchbuild/utils/schema.py | 77 ++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/benchbuild/utils/schema.py b/benchbuild/utils/schema.py index 8b1263d77..9c4eec3fa 100644 --- a/benchbuild/utils/schema.py +++ b/benchbuild/utils/schema.py @@ -78,10 +78,13 @@ def metadata(): return BASE.metadata +ET = tp.TypeVar('ET', bound=sa.exc.SQLAlchemyError) + + def exceptions( error_is_fatal: bool = True, - error_messages: tp.Optional[tp.Dict[Exception, str]] = None -) -> tp.Callable: + error_messages: tp.Optional[tp.Dict[tp.Type[ET], str]] = None +) -> tp.Callable[..., tp.Any]: """ Handle SQLAlchemy exceptions in a sane way. @@ -94,18 +97,20 @@ def exceptions( customized error message. """ - def exception_decorator(func): + def exception_decorator( + func: tp.Callable[..., tp.Any] + ) -> tp.Callable[..., tp.Any]: nonlocal error_messages @functools.wraps(func) - def exc_wrapper(*args, **kwargs): + def exc_wrapper(*args: tp.Any, **kwargs: tp.Any) -> tp.Optional[tp.Any]: nonlocal error_messages try: result = func(*args, **kwargs) except sa.exc.SQLAlchemyError as err: result = None details = None - err_type = err.__class__ + err_type = type(err) if error_messages and err_type in error_messages: details = error_messages[err_type] if details: @@ -131,10 +136,12 @@ class GUID(TypeDecorator): CHAR(32), storing as stringified hex values. """ impl = CHAR - as_uuid = False - cache_ok = False + as_uuid: bool = False + cache_ok: bool = False - def __init__(self, *args, as_uuid=False, **kwargs): + def __init__( + self, *args: tp.Any, as_uuid: bool = False, **kwargs: tp.Any + ) -> None: self.as_uuid = as_uuid super().__init__(*args, **kwargs) @@ -143,10 +150,12 @@ def load_dialect_impl(self, dialect): return dialect.type_descriptor(UUID(as_uuid=self.as_uuid)) return dialect.type_descriptor(CHAR(32)) - def process_bind_param(self, value, dialect): + def process_bind_param(self, value, dialect) -> str: return str(value) - def process_result_value(self, value, dialect): + def process_result_value( + self, value: tp.Union[uuid.UUID, str], dialect + ) -> uuid.UUID: if isinstance(value, uuid.UUID): return value @@ -180,32 +189,32 @@ class Run(BASE): end = Column(DateTime(timezone=False)) status = Column(Enum('completed', 'running', 'failed', name="run_state")) - metrics = sa.orm.relationship( + metrics: Metric = sa.orm.relationship( "Metric", cascade="all, delete-orphan", passive_deletes=True, passive_updates=True ) - logs = sa.orm.relationship( + logs: RunLog = sa.orm.relationship( "RunLog", cascade="all, delete-orphan", passive_deletes=True, passive_updates=True ) - stored_data = sa.orm.relationship( + stored_data: Metadata = sa.orm.relationship( "Metadata", cascade="all, delete-orphan", passive_deletes=True, passive_updates=True ) - configurations = sa.orm.relationship( + configurations: Config = sa.orm.relationship( "Config", cascade="all, delete-orphan", passive_deletes=True, passive_updates=True ) - def __repr__(self): + def __repr__(self) -> str: return ( f'' ) @@ -239,20 +248,20 @@ class Experiment(BASE): begin = Column(DateTime(timezone=False)) end = Column(DateTime(timezone=False)) - runs = sa.orm.relationship( + runs: Run = sa.orm.relationship( "Run", cascade="all, delete-orphan", passive_deletes=True, passive_updates=True ) - run_groups = sa.orm.relationship( + run_groups: RunGroup = sa.orm.relationship( "RunGroup", cascade="all, delete-orphan", passive_deletes=True, passive_updates=True ) - def __repr__(self): + def __repr__(self) -> str: return f'' @@ -268,14 +277,14 @@ class Project(BASE): group_name = Column(String, primary_key=True) version = Column(String) - runs = sa.orm.relationship( + runs: Run = sa.orm.relationship( "Run", cascade="all, delete-orphan", passive_deletes=True, passive_updates=True ) - def __repr__(self): + def __repr__(self) -> str: return f'' @@ -293,7 +302,7 @@ class Metric(BASE): primary_key=True ) - def __repr__(self): + def __repr__(self) -> str: return f'{self.name} - {self.value}' @@ -361,7 +370,7 @@ class Config(BASE): value = Column(String) -def needed_schema(connection, meta): +def needed_schema(connection, meta) -> bool: try: meta.create_all(connection, checkfirst=False) except sa.exc.CompileError as cerr: @@ -382,7 +391,7 @@ def needed_schema(connection, meta): return True -def get_version_data(): +def get_version_data() -> tp.Tuple[migrate.VerNum, migrate.VerNum]: """Retreive migration information.""" connect_str = str(settings.CFG["db"]["connect_string"]) repo_url = path.template_path("../db/") @@ -397,7 +406,7 @@ def get_version_data(): ) } ) -def enforce_versioning(force=False): +def enforce_versioning(force: bool = False) -> tp.Optional[str]: """Install versioning on the db.""" connect_str, repo_url = get_version_data() LOG.debug("Your database uses an unversioned benchbuild schema.") @@ -406,12 +415,12 @@ def enforce_versioning(force=False): ): LOG.error("User declined schema versioning.") return None - repo_version = migrate.version(repo_url, url=connect_str) + repo_version: str = migrate.version(repo_url, url=connect_str) migrate.version_control(connect_str, repo_url, version=repo_version) return repo_version -def setup_versioning(): +def setup_versioning() -> tp.Tuple[migrate.VerNum, migrate.VerNum]: connect_str, repo_url = get_version_data() repo_version = migrate.version(repo_url, url=connect_str) db_version = None @@ -434,7 +443,9 @@ def setup_versioning(): " Base schema version diverged from the expected structure." } ) -def maybe_update_db(repo_version, db_version): +def maybe_update_db( + repo_version: str, db_version: tp.Optional[str] = None +) -> None: if db_version is None: return if db_version == repo_version: @@ -463,7 +474,7 @@ def maybe_update_db(repo_version, db_version): class SessionManager: - def connect_engine(self): + def connect_engine(self) -> bool: """ Establish a connection to the database. @@ -482,7 +493,7 @@ def connect_engine(self): ) return False - def configure_engine(self): + def configure_engine(self) -> bool: """ Configure the databse connection. @@ -503,7 +514,7 @@ def configure_engine(self): "Connect string contained an invalid backend." } ) - def __init__(self): + def __init__(self) -> None: self.__test_mode = bool(settings.CFG['db']['rollback']) self.engine = create_engine(str(settings.CFG["db"]["connect_string"])) @@ -525,7 +536,7 @@ def __init__(self): def get(self): return sessionmaker(bind=self.connection) - def __del__(self): + def __del__(self) -> None: if hasattr(self, '__transaction') and self.__transaction: self.__transaction.rollback() @@ -547,10 +558,10 @@ def __lazy_session_wrapped() -> CanCommit: return __lazy_session_wrapped -Session = __lazy_session__() +Session: tp.Callable[[], CanCommit] = __lazy_session__() -def init_functions(connection): +def init_functions(connection) -> None: """Initialize all SQL functions in the database.""" if settings.CFG["db"]["create_functions"]: print("Refreshing SQL functions...") From 9703bbd7a9b1b115d3db60a966e19c826c96a694 Mon Sep 17 00:00:00 2001 From: Andreas Simbuerger Date: Wed, 26 Jan 2022 09:52:39 +0100 Subject: [PATCH 32/32] fix(utils/run): tp.Protocol -> Protocol (3.7) --- benchbuild/utils/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchbuild/utils/run.py b/benchbuild/utils/run.py index 7c95d12ef..1178ebd10 100644 --- a/benchbuild/utils/run.py +++ b/benchbuild/utils/run.py @@ -378,7 +378,7 @@ def with_env_recursive(cmd: BaseCommand, **envvars: str) -> BaseCommand: return cmd -class HasBuilddir(tp.Protocol): +class HasBuilddir(Protocol): builddir: local.path