post needs a preceding ') or a_src != a_post_src:
- raise ColourError('A post (expanded %r from %r)' % (a, a_src,))
+ raise ColourError(f'A post (expanded {a!r} from {a_src!r})')
literal_post += a_post[1:]
pre.append(a)
continue
@@ -228,12 +219,12 @@ def pre_post(self, *attrs, **kw):
raise ValueError()
part = (prefix, '5', str(idx))
except (ValueError, AssertionError):
- raise ColourError('Bad colour spec %r (from %r)' % (a, a_src,))
+ raise ColourError(f'Bad colour spec {a!r} (from {a_src!r})')
pre.append(':'.join(part))
post.add(default)
else:
if want not in self._all:
- raise ColourError('Unknown colour/attr %r (from %r)' % (a, a_src,))
+ raise ColourError(f'Unknown colour/attr {a!r} (from {a_src!r})')
pre.append(self._all[want])
post.add(self._all.get('NOT' + want, default))
pre = ''.join(self._literal_split(pre))
@@ -252,7 +243,7 @@ def __call__(self, value, *attrs, **kw):
pre, post = self.pre_post(*attrs, **kw)
if isinstance(value, bytes):
return b'%s%s%s' % (pre.encode('utf-8'), value, post.encode('utf-8'),)
- return '%s%s%s' % (pre, value, post,)
+ return f'{pre}{value}{post}'
colour = Colour()
colour.configure_from_environ()
diff --git a/accelerator/compat.py b/accelerator/compat.py
index ce8a6a89..86b7872b 100644
--- a/accelerator/compat.py
+++ b/accelerator/compat.py
@@ -20,10 +20,6 @@
# a few things that differ between python2 and python3
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
import sys
try:
from setproctitle import setproctitle as _setproctitle, getproctitle
@@ -33,131 +29,41 @@
except ImportError:
def _setproctitle(title): pass
-if sys.version_info[0] == 2:
- PY2 = True
- PY3 = False
- import __builtin__ as builtins
- import cPickle as pickle
- FileNotFoundError = (builtins.OSError, builtins.IOError)
- PermissionError = builtins.IOError
- from urllib import quote as url_quote, quote_plus, unquote_plus, urlencode
- from urllib2 import urlopen, Request, URLError, HTTPError
- from itertools import izip, izip_longest, imap, ifilter
- from Queue import Queue, Full as QueueFull, Empty as QueueEmpty
- import selectors2 as selectors
- try:
- from monotonic import monotonic
- except ImportError:
- from time import time as monotonic
- from types import NoneType
- str_types = (str, unicode,)
- int_types = (int, long,)
- num_types = (int, float, long,)
- unicode = builtins.unicode
- long = builtins.long
- def iterkeys(d):
- return d.iterkeys()
- def itervalues(d):
- return d.itervalues()
- def iteritems(d):
- return d.iteritems()
- from io import open
- def getarglist(func):
- from inspect import getargspec
- return getargspec(func).args
- def terminal_size():
- from termios import TIOCGWINSZ
- import struct
- from fcntl import ioctl
- from collections import namedtuple
- from os import environ
- def ifgood(name):
- try:
- v = int(environ[name])
- if v > 0:
- return v
- except (KeyError, ValueError):
- pass
- lines, columns = ifgood('LINES'), ifgood('COLUMNS')
- if not lines or not columns:
- try:
- fb_lines, fb_columns, _, _ = struct.unpack('HHHH', ioctl(0, TIOCGWINSZ, struct.pack('HHHH', 0, 0, 0, 0)))
- except Exception:
- fb_lines, fb_columns = 24, 80
- return namedtuple('terminal_size', 'columns lines')(columns or fb_columns, lines or fb_lines)
- def shell_quote(v):
- ok = set('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789,.:/+-_')
- if any(c not in ok for c in v):
- return "'%s'" % (v.replace("'", "'\"'\"'"),)
- else:
- return v or "''"
- import datetime
- _timedelta_0 = datetime.timedelta(0)
- class timezone(datetime.tzinfo):
- __slots__ = ()
- def __new__(cls, offset, name=None):
- if UTC is None: # when constructing the singleton
- return datetime.tzinfo.__new__(cls)
- if name is not None or offset != _timedelta_0:
- raise ValueError("This python 2 compat class only supports UTC.")
- return UTC
- def dst(self, dt):
- return _timedelta_0
- def utcoffset(self, dt):
- return _timedelta_0
- def tzname(self, dt):
- return b'UTC'
- def __reduce__(self):
- return _utc_pickle
- def __repr__(self):
- return b'accelerator.compat.UTC'
- def __str__(self):
- return b'UTC'
- UTC = None
- UTC = timezone(None)
- timezone.__module__ = b'datetime' # so it pickles the way we want
- datetime.timezone = timezone
- _utc_pickle = (timezone, (_timedelta_0,)) # same thing the python 3 version pickles as
- del timezone
- del datetime
-else:
- PY2 = False
- PY3 = True
- import builtins
- FileNotFoundError = builtins.FileNotFoundError
- PermissionError = builtins.PermissionError
- import pickle
- from urllib.parse import quote as url_quote, quote_plus, unquote_plus, urlencode
- from urllib.request import urlopen, Request
- from urllib.error import URLError, HTTPError
- izip = zip
- from itertools import zip_longest as izip_longest
- imap = map
- ifilter = filter
- from queue import Queue, Full as QueueFull, Empty as QueueEmpty
- import selectors
- from time import monotonic
- NoneType = type(None)
- str_types = (str,)
- int_types = (int,)
- num_types = (int, float,)
- unicode = str
- open = builtins.open
- long = int
- def iterkeys(d):
- return iter(d.keys())
- def itervalues(d):
- return iter(d.values())
- def iteritems(d):
- return iter(d.items())
- def getarglist(func):
- from inspect import getfullargspec
- return getfullargspec(func).args
- from shutil import get_terminal_size as terminal_size
- from shlex import quote as shell_quote
- import datetime
- UTC = datetime.timezone.utc
- del datetime
+import builtins
+FileNotFoundError = builtins.FileNotFoundError
+PermissionError = builtins.PermissionError
+import pickle
+from urllib.parse import quote as url_quote, quote_plus, unquote_plus, urlencode
+from urllib.request import urlopen, Request
+from urllib.error import URLError, HTTPError
+izip = zip
+from itertools import zip_longest as izip_longest
+imap = map
+ifilter = filter
+from queue import Queue, Full as QueueFull, Empty as QueueEmpty
+import selectors
+from time import monotonic
+NoneType = type(None)
+str_types = (str,)
+int_types = (int,)
+num_types = (int, float,)
+unicode = str
+open = builtins.open
+long = int
+def iterkeys(d):
+ return iter(d.keys())
+def itervalues(d):
+ return iter(d.values())
+def iteritems(d):
+ return iter(d.items())
+def getarglist(func):
+ from inspect import getfullargspec
+ return getfullargspec(func).args
+from shutil import get_terminal_size as terminal_size
+from shlex import quote as shell_quote
+import datetime
+UTC = datetime.timezone.utc
+del datetime
def first_value(d):
return next(itervalues(d) if isinstance(d, dict) else iter(d))
@@ -170,30 +76,25 @@ def uni(s):
return s.decode('utf-8')
except UnicodeDecodeError:
return s.decode('iso-8859-1')
- return unicode(s)
+ return str(s)
def url_quote_more(s):
return quote_plus(s).replace('+', '%20')
-if sys.version_info < (3, 6):
- fmt_num = '{:n}'.format
-else:
- def fmt_num(num):
- if isinstance(num, float):
- return '{:_.6g}'.format(num)
- else:
- return '{:_}'.format(num)
+def fmt_num(num):
+ if isinstance(num, float):
+ return f'{num:_.6g}'
+ else:
+ return f'{num:_}'
# This is used in the method launcher to set different titles for each
# phase/slice. You can use it in the method to override that if you want.
def setproctitle(title):
from accelerator import g
if hasattr(g, 'params'):
- title = '%s %s (%s)' % (g.job, uni(title), g.params.method,)
+ title = f'{g.job} {uni(title)} ({g.params.method})'
elif hasattr(g, 'job'):
- title = '%s %s' % (g.job, uni(title),)
+ title = f'{g.job} {uni(title)}'
else:
title = uni(title)
- if PY2:
- title = title.encode('utf-8')
_setproctitle(title)
diff --git a/accelerator/configfile.py b/accelerator/configfile.py
index 555c41af..7de507a1 100644
--- a/accelerator/configfile.py
+++ b/accelerator/configfile.py
@@ -19,14 +19,11 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-
import re
import os
import shlex
-from accelerator.compat import url_quote_more, open
+from accelerator.compat import url_quote_more
from accelerator.extras import DotDict
@@ -86,7 +83,7 @@ class _E(Exception):
def parse_package(val):
if len(val) == 2:
if val[1] != 'auto-discover':
- raise _E('Either no option or "auto-discover" for package %r.' % (val[0],))
+ raise _E(f'Either no option or "auto-discover" for package {val[0]!r}.')
else:
return (val[0], True)
return (val[0], False)
@@ -98,7 +95,7 @@ def check_interpreter(val):
if val[0] == 'DEFAULT':
raise _E("Don't override DEFAULT interpreter")
if not os.path.isfile(val[1]):
- raise _E('%r does not exist' % (val,))
+ raise _E(f'{val!r} does not exist')
def check_workdirs(val):
name, path = val
if val in cfg['workdirs']:
@@ -110,11 +107,11 @@ def check_workdirs(val):
# in your config, and have it still work for the foo and bar users.
return
if name in (v[0] for v in cfg['workdirs']):
- raise _E('Workdir %s redefined' % (name,))
+ raise _E(f'Workdir {name} redefined')
if path in (v[1] for v in cfg['workdirs']):
- raise _E('Workdir path %r re-used' % (path,))
+ raise _E(f'Workdir path {path!r} re-used')
if project_directory == path or project_directory.startswith(path + '/'):
- raise _E('project directory (%r) is under workdir %s (%r)' % (project_directory, name, path))
+ raise _E(f'project directory ({project_directory!r}) is under workdir {name} ({path!r})')
def resolve_urd(val):
orig_val = val
@@ -125,7 +122,7 @@ def resolve_urd(val):
else:
val = [val[1]]
if len(val) != 1:
- raise _E("urd takes 1 or 2 values (expected %s, got %r)" % (' '.join(parsers['urd'][0]), orig_val))
+ raise _E(f"urd takes 1 or 2 values (expected {' '.join(parsers['urd'][0])}, got {orig_val!r})")
return is_local, resolve_listen(val[0])
parsers = {
@@ -158,7 +155,7 @@ def parse(handle):
raise _E('Expected a ":"')
key, val = line.split(':', 1)
if key not in known:
- raise _E('Unknown key %r' % (key,))
+ raise _E(f'Unknown key {key!r}')
else:
if not key:
raise _E('First line indented')
@@ -169,7 +166,7 @@ def parse(handle):
def just_project_directory(key, val):
if key == 'project directory':
if len(val) != 1:
- raise _E("%s takes a single value path (maybe you meant to quote it?)" % (key,))
+ raise _E(f"{key} takes a single value path (maybe you meant to quote it?)")
project_directory[0] = val[0]
def everything(key, val):
if key in parsers:
@@ -182,23 +179,23 @@ def everything(key, val):
want_count_str = str(want_count[0])
if len(val) not in want_count:
if len(args) == 1:
- raise _E("%s takes a single value %s (maybe you meant to quote it?)" % (key, args[0]))
+ raise _E(f"{key} takes a single value {args[0]} (maybe you meant to quote it?)")
else:
- raise _E("%s takes %s values (expected %s, got %r)" % (key, want_count_str, ' '.join(args), val))
+ raise _E(f"{key} takes {want_count_str} values (expected {' '.join(args)}, got {val!r})")
if len(args) == 1:
val = val[0]
val = p(val)
elif len(val) == 1:
val = val[0]
else:
- raise _E("%s takes a single value (maybe you meant to quote it?)" % (key,))
+ raise _E(f"{key} takes a single value (maybe you meant to quote it?)")
if key in checkers:
checkers[key](val)
if key in multivalued:
cfg[key].append(val)
else:
if key in cfg:
- raise _E("%r doesn't take multiple values" % (key,))
+ raise _E(f"{key!r} doesn't take multiple values")
cfg[key] = val
try:
@@ -215,7 +212,7 @@ def everything(key, val):
if not cfg[req]:
missing.add(req)
if missing:
- raise _E('Missing required keys %r' % (missing,))
+ raise _E(f'Missing required keys {missing!r}')
# Reformat result a bit so the new format doesn't require code changes all over the place.
rename = {
@@ -240,10 +237,10 @@ def everything(key, val):
res.result_directory = fixpath('')
res.workdirs = dict(res.workdirs)
if res.target_workdir not in res.workdirs:
- raise _E('target workdir %r not in defined workdirs %r' % (res.target_workdir, set(res.workdirs),))
+ raise _E(f'target workdir {res.target_workdir!r} not in defined workdirs {set(res.workdirs)!r}')
res.interpreters = dict(res.interpreters)
for exe in res.interpreters.values():
- assert os.path.exists(exe), 'Executable %r does not exist.' % (exe,)
+ assert os.path.exists(exe), f'Executable {exe!r} does not exist.'
res.listen, res.url = fixup_listen(res.project_directory, res.listen)
if res.get('urd'):
res.urd_local, listen = res.urd
@@ -254,9 +251,9 @@ def everything(key, val):
res.method_directories = dict(res.method_directories)
except _E as e:
if lineno[0] is None:
- prefix = 'Error in %s:\n' % (filename,)
+ prefix = f'Error in {filename}:\n'
else:
- prefix = 'Error on line %d of %s:\n' % (lineno[0], filename,)
+ prefix = f'Error on line {lineno[0]} of {filename}:\n'
raise UserError(prefix + e.args[0])
res.config_filename = os.path.realpath(filename)
diff --git a/accelerator/control.py b/accelerator/control.py
index 8ed8b42f..7fd3cd37 100644
--- a/accelerator/control.py
+++ b/accelerator/control.py
@@ -19,9 +19,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-
from threading import Thread
import multiprocessing
import signal
@@ -76,7 +73,7 @@ def __init__(self, config, options, server_url):
def _update_methods(self, _override_py_exe=None):
print('Update methods')
if _override_py_exe:
- assert exists(_override_py_exe), 'Executable %r does not exist.' % (_override_py_exe,)
+ assert exists(_override_py_exe), f'Executable {_override_py_exe!r} does not exist.'
config = DotDict(self.config)
# Doesn't override the automatic version.number names.
config.interpreters = {
@@ -166,7 +163,7 @@ def update(name):
t_l.append(t)
for t in t_l:
t.join()
- assert not failed, "%s failed to update" % (', '.join(failed),)
+ assert not failed, f"{', '.join(failed)} failed to update"
finally:
pool.close()
self.DataBase._update_finish(self.Methods.hash)
@@ -176,7 +173,7 @@ def initialise_jobs(self, setup, workdir=None):
""" Update database, check deps, create jobids. """
ws = workdir or self.target_workdir
if ws not in self.workspaces:
- raise BuildError("Workdir %s does not exist" % (ws,))
+ raise BuildError(f"Workdir {ws} does not exist")
return dependency.initialise_jobs(
setup,
self.workspaces[ws],
diff --git a/accelerator/database.py b/accelerator/database.py
index ad1668a6..46220182 100644
--- a/accelerator/database.py
+++ b/accelerator/database.py
@@ -19,9 +19,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-
from collections import defaultdict
from collections import namedtuple
from traceback import print_exc
@@ -106,7 +103,7 @@ def add_single_jobid(self, jobid):
def _update_workspace(self, WorkSpace, pool, verbose=False):
"""Insert all items in WorkSpace in database (call update_finish too)"""
if verbose:
- print("DATABASE: update for \"%s\"" % WorkSpace.name)
+ print(f"DATABASE: update for \"{WorkSpace.name}\"")
filesystem_jobids = WorkSpace.valid_jobids
self._fsjid.update(filesystem_jobids)
if verbose > 1:
@@ -116,7 +113,7 @@ def _update_workspace(self, WorkSpace, pool, verbose=False):
if new_jobids:
_paramsdict.update(pool.imap_unordered(_get_params, new_jobids, chunksize=64))
if verbose:
- print("DATABASE: Database \"%s\" contains %d potential items" % (WorkSpace.name, len(filesystem_jobids), ))
+ print(f"DATABASE: Database \"{WorkSpace.name}\" contains {len(filesystem_jobids)} potential items")
def _update_finish(self, dict_of_hashes, verbose=False):
"""Filters in-use database on valid hashes.
@@ -175,8 +172,8 @@ def _update_finish(self, dict_of_hashes, verbose=False):
l.sort(key=lambda jid: _paramsdict[jid][0].starttime)
if verbose:
if discarded_due_to_hash_list:
- print("DATABASE: discarding due to unknown hash: %s" % ', '.join(discarded_due_to_hash_list))
- print("DATABASE: Full database contains %d items" % (sum(len(v) for v in itervalues(self.db_by_method)),))
+ print(f"DATABASE: discarding due to unknown hash: {', '.join(discarded_due_to_hash_list)}")
+ print(f"DATABASE: Full database contains {sum((len(v) for v in itervalues(self.db_by_method)))} items")
def match_complex(self, reqlist):
for method, uid, opttuple in reqlist:
diff --git a/accelerator/dataset.py b/accelerator/dataset.py
index 0c2ee9ca..d9881315 100644
--- a/accelerator/dataset.py
+++ b/accelerator/dataset.py
@@ -20,10 +20,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
import os
from keyword import kwlist
from collections import namedtuple, Counter
@@ -34,8 +30,8 @@
from math import isnan
import datetime
-from accelerator.compat import unicode, uni, ifilter, imap, iteritems, PY2
-from accelerator.compat import builtins, open, getarglist, izip, izip_longest
+from accelerator.compat import uni, ifilter, imap, iteritems
+from accelerator.compat import builtins, getarglist, izip, izip_longest
from accelerator.compat import str_types, int_types, FileNotFoundError
from accelerator import blob
@@ -91,7 +87,7 @@
def _clean_name(n, seen_n):
n = ''.join(c if c.isalnum() else '_' for c in n)
if not n or n[0].isdigit():
- n = '_' + n
+ n = f'_{n}'
while n in seen_n or iskeyword(n):
n += '_'
seen_n.add(n)
@@ -104,7 +100,7 @@ def _dsid(t):
jid, name = t
if not jid:
return None
- t = '%s/%s' % (jid.split('/')[0], uni(name) or 'default')
+ t = f"{jid.split('/')[0]}/{uni(name) or 'default'}"
elif isinstance(t, DatasetWriter):
t = t.ds_name
if '/' not in t:
@@ -138,13 +134,13 @@ def __new__(cls, type, backing_type, name, location, min, max, offsets):
none_support = not backing_type.startswith('bits')
return _DatasetColumn_3_1(type, backing_type, name, location, min, max, offsets, none_support)
-class _New_dataset_marker(unicode): pass
+class _New_dataset_marker(str): pass
_new_dataset_marker = _New_dataset_marker('new')
_no_override = object()
_ds_cache = {}
def _ds_load(obj):
- n = unicode(obj)
+ n = str(obj)
if n not in _ds_cache:
fn = obj.job.filename(obj._name('pickle'))
if not os.path.exists(fn):
@@ -152,17 +148,17 @@ def _ds_load(obj):
try:
dslist = obj.job.datasets
if not dslist:
- extra = ' (%s contains no datasets)' % (obj.job,)
+ extra = f' ({obj.job} contains no datasets)'
elif len(dslist) == 1:
- extra = ' (did you mean %s?)' % (dslist[0].quoted,)
+ extra = f' (did you mean {dslist[0].quoted}?)'
elif len(dslist) < 5:
- extra = ' (did you mean one of [%s]?)' % (', '.join(ds.quoted for ds in dslist),)
+ extra = f" (did you mean one of [{', '.join((ds.quoted for ds in dslist))}]?)"
else:
- extra = ' (job contains %d other datasets, use "ax ds -l %s" to list them)' % (len(dslist), obj.job,)
+ extra = f' (job contains {len(dslist)} other datasets, use "ax ds -l {obj.job}" to list them)'
except Exception:
# We don't want to accidentally give an unrelated exception
pass
- raise NoSuchDatasetError('Dataset %s does not exist%s' % (obj.quoted, extra,))
+ raise NoSuchDatasetError(f'Dataset {obj.quoted} does not exist{extra}')
_ds_cache[n] = blob.load(fn)
_ds_cache.update(_ds_cache[n].get('cache', ()))
return _ds_cache[n]
@@ -173,13 +169,13 @@ def _namechk(name):
_dummy_iter = iter(())
_dir_name_blacklist = list(range(32)) + [37, 47] # unprintable and '%/'
-_dir_name_blacklist = {chr(c): '\\x%02x' % (c,) for c in _dir_name_blacklist}
+_dir_name_blacklist = {chr(c): f'\\x{c:02x}' for c in _dir_name_blacklist}
_dir_name_blacklist['\\'] = '\\\\' # make sure names can't collide
def _fs_name(name):
- return 'DS/' + ''.join(_dir_name_blacklist.get(c, c) for c in name)
+ return f'DS/{"".join(_dir_name_blacklist.get(c, c) for c in name)}'
-class Dataset(unicode):
+class Dataset(str):
"""
Represents a dataset. Is also a string 'jobid/name', or just 'jobid' if
name is 'default' (for better backwards compatibility).
@@ -193,7 +189,7 @@ class Dataset(unicode):
You can also pass jobid={jid: dsname} to resolve dsname from the datasets
passed to jid. This gives NoDataset if that option was unset.
- These decay to a (unicode) string when pickled.
+ These decay to a string when pickled.
"""
__slots__ = ('name', 'quoted', 'job', 'fs_name', '_data', '_cache', '_job_version',)
@@ -226,10 +222,10 @@ def __new__(cls, jobid, name=None):
if name == 'default':
fullname = job
else:
- fullname = '%s/%s' % (job, name,)
- obj = unicode.__new__(cls, fullname)
+ fullname = f'{job}/{name}'
+ obj = str.__new__(cls, fullname)
obj.name = name
- obj.quoted = quote('%s/%s' % (job, name,))
+ obj.quoted = quote(f'{job}/{name}')
obj._job_version = 4 # hopefully
obj.fs_name = _fs_name(obj.name)
if jobid is _new_dataset_marker:
@@ -253,14 +249,14 @@ def __new__(cls, jobid, name=None):
obj.fs_name = obj.name
obj._data = DotDict(_ds_load(obj))
if obj._data.version[0] != 3:
- raise DatasetError("%s/%s: Unsupported dataset pickle version %r" % (jobid, name, obj._data.version,))
+ raise DatasetError(f"{jobid}/{name}: Unsupported dataset pickle version {obj._data.version!r}")
obj._data.columns = dict(obj._data.columns)
obj._cache = {}
return obj
# Look like a string after pickling
def __reduce__(self):
- return unicode, (unicode(self),)
+ return str, (str(self),)
@property
def columns(self):
@@ -330,7 +326,7 @@ def link_to_here(self, name='default', column_filter=None, rename=None, override
You can change the filename too, or clear it by setting ''.
"""
if name in _datasetwriters or os.path.exists(_fs_name(name) + '.p'):
- raise DatasetUsageError('Duplicate dataset name "%s"' % (name,))
+ raise DatasetUsageError(f'Duplicate dataset name "{name}"')
d = Dataset(self)
if rename:
renamed = {}
@@ -338,12 +334,12 @@ def link_to_here(self, name='default', column_filter=None, rename=None, override
rename_discarded = set()
for k, v in rename.items():
if k not in d._data.columns:
- raise DatasetUsageError("Renamed column %r not in dataset" % (k,))
+ raise DatasetUsageError(f"Renamed column {k!r} not in dataset")
if v is None:
rename_discarded.add(k)
continue
if v in renamed_src:
- raise DatasetUsageError("Both %r and %r renamed to %r" % (k, renamed_src[v], v,))
+ raise DatasetUsageError(f"Both {k!r} and {renamed_src[v]!r} renamed to {v!r}")
renamed[v] = d._data.columns.pop(k)
renamed_src[v] = k
d._data.columns.update(renamed)
@@ -363,7 +359,7 @@ def link_to_here(self, name='default', column_filter=None, rename=None, override
filtered_columns = {k: v for k, v in d._data.columns.items() if k in column_filter}
left_over = column_filter - set(filtered_columns)
if left_over:
- raise DatasetUsageError("Columns in filter not available in dataset: %r" % (left_over,))
+ raise DatasetUsageError(f"Columns in filter not available in dataset: {left_over!r}")
if not filtered_columns:
raise DatasetUsageError("Filter produced no desired columns.")
d._data.columns = filtered_columns
@@ -375,7 +371,7 @@ def link_to_here(self, name='default', column_filter=None, rename=None, override
Dataset(override_previous)
d._data.previous = override_previous
d._update_caches()
- d._data.parent = '%s/%s' % (d.job, d.name,)
+ d._data.parent = f'{d.job}/{d.name}'
if filename is not None:
d._data.filename = filename or None
d.job = job
@@ -401,9 +397,9 @@ def merge(self, other, name='default', previous=None, allow_unrelated=False):
else:
previous = None
if self == other:
- raise DatasetUsageError("Can't merge with myself (%s)" % (other.quoted,))
+ raise DatasetUsageError(f"Can't merge with myself ({other.quoted})")
if self.lines != other.lines:
- raise DatasetUsageError("%s and %s don't have the same line counts" % (self.quoted, other.quoted,))
+ raise DatasetUsageError(f"{self.quoted} and {other.quoted} don't have the same line counts")
if other.hashlabel is not None:
new_ds._data.hashlabel = other.hashlabel
elif self.hashlabel in other.columns:
@@ -422,7 +418,7 @@ def parents(ds, tips):
return tips
related = parents(self, set()) & parents(other, set())
if not related:
- raise DatasetUsageError("%s and %s have no common ancenstors, set allow_unrelated to allow this" % (self.quoted, other.quoted,))
+ raise DatasetUsageError(f"{self.quoted} and {other.quoted} have no common ancenstors, set allow_unrelated to allow this")
new_ds.job = job
new_ds.name = name
new_ds.fs_name = _fs_name(name)
@@ -442,7 +438,7 @@ def _column_iterator(self, sliceno, col, _type=None, _line_report=None, **kw):
if not _type:
_type = dc.type
if _type not in _type2iter:
- msg = 'Unsupported column type %r on %r in %s' % (_type, col, self.quoted,)
+ msg = f'Unsupported column type {_type!r} on {col!r} in {self.quoted}'
if _type.startswith('bits'):
msg += ". If you need to use this column, please install the 2022.8.4.dev1 release and use the dataset_unbits methods to convert it."
raise DatasetError(msg)
@@ -477,7 +473,7 @@ def _iterator(self, sliceno, columns=None, copy_mode=False, _line_report=None):
else:
not_found.append(col)
if not_found:
- raise DatasetError("Columns %r not found in %s/%s" % (not_found, self.job, self.name,))
+ raise DatasetError(f"Columns {not_found!r} not found in {self.job}/{self.name}")
return res
def _hashfilter(self, sliceno, hashlabel, it):
@@ -622,7 +618,7 @@ def iterate_list(sliceno, columns, datasets, range=None, sloppy_range=False, has
need_types = {col: datasets[0].columns[col].type for col in columns}
for d in datasets:
for col, t in need_types.items():
- assert d.columns[col].type == t, "%s column %s has type %s, not %s" % (d, col, d.columns[col].type, t,)
+ assert d.columns[col].type == t, f"{d} column {col} has type {d.columns[col].type}, not {t}"
to_iter = []
if range:
if len(range) != 1:
@@ -649,21 +645,21 @@ def iterate_list(sliceno, columns, datasets, range=None, sloppy_range=False, has
total_lines = sum(sum(d.lines) for d in datasets)
if slice.start < 0:
if -slice.start > total_lines:
- raise DatasetUsageError("Wanted last %d lines, but only %d lines available" % (-slice.start, total_lines,))
+ raise DatasetUsageError(f"Wanted last {-slice.start} lines, but only {total_lines} lines available")
slice = builtins.slice(total_lines + slice.start, slice.stop, slice.step)
if (slice.stop or 0) < 0:
if -slice.stop > total_lines:
- raise DatasetUsageError("Wanted to stop %d lines before end, but only %d lines available" % (-slice.stop, total_lines,))
+ raise DatasetUsageError(f"Wanted to stop {-slice.stop} lines before end, but only {total_lines} lines available")
slice = builtins.slice(slice.start, total_lines + slice.stop, slice.step)
if slice.start > total_lines:
- raise DatasetUsageError("Wanted to skip %d lines, but only %d lines available" % (slice.start, total_lines,))
+ raise DatasetUsageError(f"Wanted to skip {slice.start} lines, but only {total_lines} lines available")
if (slice.stop or 0) > total_lines:
- raise DatasetUsageError("Wanted to stop after %d lines, but only %d lines available" % (slice.stop, total_lines,))
+ raise DatasetUsageError(f"Wanted to stop after {slice.stop} lines, but only {total_lines} lines available")
if slice.start == total_lines:
return iter(())
if slice.stop is not None:
if slice.stop < slice.start:
- raise DatasetUsageError("Wanted %r, but start is bigger than stop" % (slice,))
+ raise DatasetUsageError(f"Wanted {slice!r}, but start is bigger than stop")
if slice.start == slice.stop:
return iter(())
def adj_slice(lines):
@@ -698,9 +694,9 @@ def adj_slice(lines):
continue
if hashlabel is not None and d.hashlabel != hashlabel:
if not rehash:
- raise DatasetUsageError("%s has hashlabel %r, not %r" % (d, d.hashlabel, hashlabel,))
+ raise DatasetUsageError(f"{d} has hashlabel {d.hashlabel!r}, not {hashlabel!r}")
if hashlabel not in d.columns:
- raise DatasetUsageError("Can't rehash %s on non-existant column %r" % (d, hashlabel,))
+ raise DatasetUsageError(f"Can't rehash {d} on non-existant column {hashlabel!r}")
rehash_on = hashlabel
else:
rehash_on = None
@@ -778,17 +774,17 @@ def _resolve_filters(columns, filters, want_tuple):
for ix, f in filters:
if f is None or f is bool:
# use value directly
- fs.append('t[%d]' % (ix,))
+ fs.append(f't[{ix}]')
else:
- n = 'f%d' % (ix,)
+ n = f'f{ix}'
arg_n.append(n)
arg_v.append(f)
- fs.append('%s(t[%d])' % (n, ix,))
- f = 'lambda t: ' + ' and '.join(fs)
+ fs.append(f'{n}(t[{ix}])')
+ f = f'lambda t: {" and ".join(fs)}'
# Add another lambda to put all fN into local variables.
# (This is faster than putting them in "locals", you get
# LOAD_DEREF instead of LOAD_GLOBAL.)
- f = 'lambda %s: %s' % (', '.join(arg_n), f)
+ f = f"lambda {', '.join(arg_n)}: {f}"
return eval(f, {}, {})(*arg_v)
else:
return filters
@@ -817,15 +813,15 @@ def _iterstatus(status_reporting, to_iter):
def fmt_dsname(d, sliceno, rehash):
if rehash is not None:
sliceno = 'REHASH'
- return '%s:%s' % (d.quoted, sliceno)
+ return f'{d.quoted}:{sliceno}'
if len(to_iter) == 1:
- msg_head = 'Iterating ' + fmt_dsname(*to_iter[0])
+ msg_head = f'Iterating {fmt_dsname(*to_iter[0])}'
def update_status(ix, d, sliceno, rehash):
pass
else:
- msg_head = 'Iterating %s to %s' % (fmt_dsname(*to_iter[0]), fmt_dsname(*to_iter[-1]),)
+ msg_head = f'Iterating {fmt_dsname(*to_iter[0])} to {fmt_dsname(*to_iter[-1])}'
def update_status(ix, d, sliceno, rehash):
- update('%s, %d/%d (%s)' % (msg_head, ix, len(to_iter), fmt_dsname(d, sliceno, rehash)))
+ update(f'{msg_head}, {ix}/{len(to_iter)} ({fmt_dsname(d, sliceno, rehash)})')
with status(msg_head) as update:
update_status._line_report = getattr(update, '_line_report', None)
yield update_status
@@ -929,7 +925,7 @@ def new(columns, filenames, compressions, lines, minmax={}, filename=None, hashl
if hashlabel is not None:
hashlabel = uni(hashlabel)
if hashlabel not in columns:
- raise DatasetUsageError("Hashlabel (%r) does not exist" % (hashlabel,))
+ raise DatasetUsageError(f"Hashlabel ({hashlabel!r}) does not exist")
res = Dataset(_new_dataset_marker, name)
res._data.lines = list(Dataset._linefixup(lines))
res._data.hashlabel = hashlabel
@@ -952,7 +948,7 @@ def append(self, columns, filenames, compressions, lines, minmax={}, filename=No
if hashlabel_override:
self._data.hashlabel = hashlabel
elif hashlabel is not None and self.hashlabel != hashlabel:
- raise DatasetUsageError("Hashlabel mismatch %r != %r" % (self.hashlabel, hashlabel,))
+ raise DatasetUsageError(f"Hashlabel mismatch {self.hashlabel!r} != {hashlabel!r}")
if self._linefixup(lines) != self.lines:
from accelerator.g import job
raise DatasetUsageError("New columns don't have the same number of lines as parent columns (trying to append %s to %s, expected %r but got %r)" % (quote('%s/%s' % (job, name,)), self.quoted, self.lines, self._linefixup(lines),))
@@ -1000,7 +996,7 @@ def _append(self, columns, filenames, compressions, minmax, filename, caption, p
if set(columns) != set(compressions):
raise DatasetUsageError("columns and compressions don't have the same keys")
if self.job and (self.job != job or self.name != name):
- self._data.parent = '%s/%s' % (self.job, self.name,)
+ self._data.parent = f'{self.job}/{self.name}'
self.job = job
self.name = name
self.fs_name = _fs_name(name)
@@ -1015,11 +1011,11 @@ def _append(self, columns, filenames, compressions, minmax, filename, caption, p
filtered_columns = {k: v for k, v in self._data.columns.items() if k in column_filter}
left_over = column_filter - set(filtered_columns)
if left_over:
- raise DatasetUsageError("Columns in filter not available in dataset: %r" % (left_over,))
+ raise DatasetUsageError(f"Columns in filter not available in dataset: {left_over!r}")
self._data.columns = filtered_columns
for n, (t, none_support) in sorted(columns.items()):
if t not in _type2iter:
- raise DatasetUsageError('Unknown type %s on column %s' % (t, n,))
+ raise DatasetUsageError(f'Unknown type {t} on column {n}')
mm = minmax.get(n, (None, None,))
t = uni(t)
self._data.columns[n] = DatasetColumn(
@@ -1048,7 +1044,7 @@ def _update_caches(self):
except NoSuchDatasetError:
j, n = self._data['previous'].split('/', 1)
if self.job == j and n in _datasetwriters:
- raise DatasetUsageError('previous dataset %r must be .finish()ed first.' % (n,))
+ raise DatasetUsageError(f'previous dataset {n!r} must be .finish()ed first.')
else:
raise
if d:
@@ -1056,7 +1052,7 @@ def _update_caches(self):
if cache_distance == 64:
cache_distance = 0
chain = self.chain(64)
- self._data['cache'] = tuple((unicode(d), d._data) for d in chain[:-1])
+ self._data['cache'] = tuple((str(d), d._data) for d in chain[:-1])
self._data['cache_distance'] = cache_distance
def _maybe_merge(self, n):
@@ -1088,7 +1084,7 @@ def getsize(sliceno):
if size:
with open(fn(sliceno), 'rb') as p_fh:
data = p_fh.read()
- assert len(data) == size, "Slice %d is %d bytes, not %d?" % (sliceno, len(data), size,)
+ assert len(data) == size, f"Slice {sliceno} is {len(data)} bytes, not {size}?"
m_fh.write(data)
offsets.append(pos)
else:
@@ -1101,7 +1097,7 @@ def getsize(sliceno):
location = c.location.split('/', 1) # the jobid might have % in it, so split it off
self._data.columns[n] = c._replace(
offsets=offsets,
- location='%s/%s' % (location[0], location[1] % ('m',)),
+ location=f"{location[0]}/{location[1] % ('m',)}",
)
def _maybe_merge_fully(self, columns):
@@ -1120,14 +1116,14 @@ def _maybe_merge_fully(self, columns):
if z > 16 * 524288: # arbitrary guess of good size
return
m_fn = self._name('merged')
- m_location = '%s/%s' % (self.job, m_fn,)
+ m_location = f'{self.job}/{m_fn}'
with open(m_fn, 'wb') as m_fh:
for n in columns:
dc = self._data.columns[n]
c_fn = self.column_filename(n)
with open(c_fn, 'rb') as c_fh:
data = c_fh.read()
- assert len(data) > max(dc.offsets), '%s is too short' % (c_fn,)
+ assert len(data) > max(dc.offsets), f'{c_fn} is too short'
os.unlink(c_fn)
pos = m_fh.tell()
m_offsets = [False if o is False else o + pos for o in dc.offsets]
@@ -1145,7 +1141,7 @@ def _name(self, thing):
assert thing == 'pickle'
return self.name + '/dataset.pickle'
else:
- return '%s.%s' % (self.fs_name, thing[0])
+ return f'{self.fs_name}.{thing[0]}'
_datasetwriters = {}
_datasets_written = []
@@ -1242,13 +1238,13 @@ def __new__(cls, columns={}, filename=None, hashlabel=None, hashlabel_override=F
from accelerator.g import running, job
if running == 'analysis':
if name not in _datasetwriters:
- raise DatasetUsageError('Dataset with name "%s" not created' % (name,))
+ raise DatasetUsageError(f'Dataset with name "{name}" not created')
if columns or filename or hashlabel or hashlabel_override or caption or previous or parent or meta_only or for_single_slice is not None:
raise DatasetUsageError("Don't specify any arguments (except optionally name) in analysis")
return _datasetwriters[name]
else:
if name in _datasetwriters or os.path.exists(_fs_name(name) + '.p'):
- raise DatasetUsageError('Duplicate dataset name "%s"' % (name,))
+ raise DatasetUsageError(f'Duplicate dataset name "{name}"')
fs_name = _fs_name(name) + '.d'
if not os.path.exists('DS'):
os.mkdir('DS')
@@ -1261,7 +1257,7 @@ def __new__(cls, columns={}, filename=None, hashlabel=None, hashlabel_override=F
obj.caption = uni(caption)
obj.previous = _dsid(previous)
obj.name = name
- obj.ds_name = '%s/%s' % (job, name,)
+ obj.ds_name = f'{job}/{name}'
obj.quoted_ds_name = quote(obj.ds_name)
obj.fs_name = fs_name
obj.columns = {}
@@ -1278,13 +1274,13 @@ def __new__(cls, columns={}, filename=None, hashlabel=None, hashlabel_override=F
if not hashlabel_override:
if obj.hashlabel is not None:
if obj.hashlabel != obj.parent.hashlabel:
- raise DatasetUsageError("Hashlabel mismatch %r != %r" % (obj.hashlabel, obj.parent.hashlabel,))
+ raise DatasetUsageError(f"Hashlabel mismatch {obj.hashlabel!r} != {obj.parent.hashlabel!r}")
elif obj.parent.hashlabel in columns:
obj.hashlabel = obj.parent.hashlabel
parent_cols = obj.parent.columns
unknown_discards = discard_columns - set(parent_cols)
if unknown_discards:
- raise DatasetUsageError("Can't discard non-existant columns %r" % (unknown_discards,))
+ raise DatasetUsageError(f"Can't discard non-existant columns {unknown_discards!r}")
obj._column_filter = set(parent_cols) - discard_columns
else:
if discard_columns:
@@ -1323,7 +1319,7 @@ def add(self, colname, coltype, default=_nodefault, none_support=_nodefault):
colname = uni(colname)
coltype = uni(coltype)
if colname in self.columns:
- raise DatasetUsageError("Column %s already exists" % (colname,))
+ raise DatasetUsageError(f"Column {colname} already exists")
try:
typed_writer(coltype) # gives error for unknown types
except ValueError as e:
@@ -1342,7 +1338,7 @@ def set_slice(self, sliceno):
from accelerator import g
if g.running == 'analysis' and self._for_single_slice != g.sliceno:
if self._for_single_slice is not None:
- raise DatasetUsageError("This writer is for slice %d" % (self._for_single_slice,))
+ raise DatasetUsageError(f"This writer is for slice {self._for_single_slice}")
else:
raise DatasetUsageError("Only use set_slice in analysis together with for_single_slice")
self._set_slice(sliceno)
@@ -1352,7 +1348,7 @@ def _set_slice(self, sliceno):
if self._started == 2:
raise DatasetUsageError("Don't use both set_slice and a split writer")
if not isinstance(sliceno, int_types) or sliceno < 0 or sliceno >= slices:
- raise DatasetUsageError("sliceno must be int in range(%d)" % (slices,))
+ raise DatasetUsageError(f"sliceno must be int in range({slices})")
self.close()
self.sliceno = sliceno
writers = self._mkwriters(sliceno)
@@ -1363,7 +1359,7 @@ def _set_slice(self, sliceno):
def column_filename(self, colname, sliceno=None):
if sliceno is None:
sliceno = self.sliceno
- return '%s/%d.%s' % (self.fs_name, sliceno, self._filenames[colname],)
+ return f'{self.fs_name}/{sliceno}.{self._filenames[colname]}'
def enable_hash_discard(self):
"""Make the write functions silently discard data that does not
@@ -1380,7 +1376,7 @@ def _mkwriters(self, sliceno, filtered=True):
if not self.columns:
raise DatasetUsageError("No columns in dataset")
if self.hashlabel is not None and self.hashlabel not in self.columns:
- raise DatasetUsageError("Hashed column (%r) missing" % (self.hashlabel,))
+ raise DatasetUsageError(f"Hashed column ({self.hashlabel!r}) missing")
self._started = 2 - filtered
if self.meta_only:
return
@@ -1389,7 +1385,7 @@ def _mkwriters(self, sliceno, filtered=True):
if self._copy_mode:
coltype = _copy_mode_overrides.get(coltype, coltype)
wt = typed_writer(coltype)
- error_extra = ' (column %s (type %s) in %s)' % (quote(colname), coltype, self.quoted_ds_name,)
+ error_extra = f' (column {quote(colname)} (type {coltype}) in {self.quoted_ds_name})'
kw = {'none_support': none_support, 'error_extra': error_extra, 'compression': 'gzip'}
if default is not _nodefault:
kw['default'] = default
@@ -1431,26 +1427,26 @@ def write_dict(values):
hix = -1
used_names = set()
names = [_clean_name(n, used_names) for n in self._order]
- w_names = [_clean_name('w%d' % (ix,), used_names) for ix in range(len(w_l))]
+ w_names = [_clean_name(f'w{ix}', used_names) for ix in range(len(w_l))]
w_d = dict(zip(w_names, w_l))
errcls = _clean_name('DatasetUsageError', used_names)
w_d[errcls] = DatasetUsageError
- f = ['def write(' + ', '.join(names) + '):']
+ f = [f'def write({", ".join(names)}):']
f_list = ['def write_list(values):']
if len(names) == 1: # only the hashlabel, no check needed
- f.append(' %s(%s)' % (w_names[0], names[0],))
- f_list.append(' %s(values[0])' % (w_names[0],))
+ f.append(f' {w_names[0]}({names[0]})')
+ f_list.append(f' {w_names[0]}(values[0])')
else:
if hl is not None:
- f.append(' if %s(%s):' % (w_names[hix], names[hix],))
- f_list.append(' if %s(values[%d]):' % (w_names[hix], hix,))
+ f.append(f' if {w_names[hix]}({names[hix]}):')
+ f_list.append(f' if {w_names[hix]}(values[{hix}]):')
for ix in range(len(names)):
if ix != hix:
- f.append(' %s(%s)' % (w_names[ix], names[ix],))
- f_list.append(' %s(values[%d])' % (w_names[ix], ix,))
+ f.append(f' {w_names[ix]}({names[ix]})')
+ f_list.append(f' {w_names[ix]}(values[{ix}])')
if hl is not None and not discard:
- f.append(' else: raise %s(%r)' % (errcls, wrong_slice_msg,))
- f_list.append(' else: raise %s(%r)' % (errcls, wrong_slice_msg,))
+ f.append(f' else: raise {errcls}({wrong_slice_msg!r})')
+ f_list.append(f' else: raise {errcls}({wrong_slice_msg!r})')
eval(compile('\n'.join(f), '', 'exec'), w_d)
self.write = w_d['write']
eval(compile('\n'.join(f_list), '', 'exec'), w_d)
@@ -1479,7 +1475,7 @@ def _mksplit(self):
raise DatasetUsageError("Don't try to use writer after .finish()ing it")
if g.running == 'analysis' and self._for_single_slice != g.sliceno:
if self._for_single_slice is not None:
- raise DatasetUsageError("This writer is for slice %d" % (self._for_single_slice,))
+ raise DatasetUsageError(f"This writer is for slice {self._for_single_slice}")
else:
raise DatasetUsageError("Only use a split writer in analysis together with for_single_slice")
if self._started == 1:
@@ -1494,7 +1490,7 @@ def key(t):
return self._order.index(t[0])
def d2l(d):
return [w.write for _, w in sorted(d.items(), key=key)]
- f_____ = ['def split(' + ', '.join(names) + '):']
+ f_____ = [f'def split({", ".join(names)}):']
f_list = ['def split_list(v):']
f_dict = ['def split_dict(d):']
from accelerator.g import slices
@@ -1520,22 +1516,22 @@ def hashwrap(v):
w_d[name_hsh] = hashwrap
else:
w_d[name_hsh] = hashfunc
- prefix = '%s = %s[%s(' % (name_w_l, name_writers, name_hsh,)
+ prefix = f'{name_w_l} = {name_writers}[{name_hsh}('
hix = self._order.index(hl)
- f_____.append('%s%s) %% %d]' % (prefix, names[hix], slices,))
- f_list.append('%sv[%d]) %% %d]' % (prefix, hix, slices,))
- f_dict.append('%sd[%r]) %% %d]' % (prefix, hl, slices,))
+ f_____.append(f'{prefix}{names[hix]}) % {slices}]')
+ f_list.append(f'{prefix}v[{hix}]) % {slices}]')
+ f_dict.append(f'{prefix}d[{hl!r}]) % {slices}]')
else:
from itertools import cycle
w_d[name_cyc] = cycle(range(slices))
- code = '%s = %s[%s(%s)]' % (name_w_l, name_writers, name_next, name_cyc,)
+ code = f'{name_w_l} = {name_writers}[{name_next}({name_cyc})]'
f_____.append(code)
f_list.append(code)
f_dict.append(code)
for ix in range(len(names)):
- f_____.append('%s[%d](%s)' % (name_w_l, ix, names[ix],))
- f_list.append('%s[%d](v[%d])' % (name_w_l, ix, ix,))
- f_dict.append('%s[%d](d[%r])' % (name_w_l, ix, self._order[ix],))
+ f_____.append(f'{name_w_l}[{ix}]({names[ix]})')
+ f_list.append(f'{name_w_l}[{ix}](v[{ix}])')
+ f_dict.append(f'{name_w_l}[{ix}](d[{self._order[ix]!r}])')
eval(compile('\n '.join(f_____), '' , 'exec'), w_d)
eval(compile('\n '.join(f_list), '', 'exec'), w_d)
eval(compile('\n '.join(f_dict), '', 'exec'), w_d)
@@ -1553,7 +1549,7 @@ def _close(self, sliceno, writers):
w.close()
len_set = set(lens.values())
if len(len_set) != 1:
- raise DatasetUsageError("Not all columns have the same linecount in slice %d: %r" % (sliceno, lens))
+ raise DatasetUsageError(f"Not all columns have the same linecount in slice {sliceno}: {lens!r}")
self._lens[sliceno] = len_set.pop()
self._minmax[sliceno] = minmax
@@ -1576,7 +1572,7 @@ def discard(self):
if self._finished:
raise DatasetUsageError("Don't try to use writer after .finish()ing it")
del _datasetwriters[self.name]
- self._finished = 'deleted dataset ' + self.name
+ self._finished = f'deleted dataset {self.name}'
from shutil import rmtree
rmtree(self.fs_name)
@@ -1597,8 +1593,6 @@ def set_compressions(self, compressions):
self._compressions = dict.fromkeys(self.columns, compressions)
else:
self._compressions.update(compressions)
- if PY2:
- self._compressions = {uni(k): uni(v) for k, v in self._compressions.items()}
def finish(self):
"""Normally you don't need to call this, but if you want to
@@ -1610,18 +1604,18 @@ def finish(self):
if self._finished:
return self._finished
if not (self._started or self.meta_only or self._lens):
- raise DatasetUsageError("DatasetWriter %r was never started (.get_split_write*() or .set_slice(), or .discard() it)" % (self.name,))
+ raise DatasetUsageError(f"DatasetWriter {self.name!r} was never started (.get_split_write*() or .set_slice(), or .discard() it)")
self.close()
if set(self._compressions) != set(self.columns):
missing = set(self.columns) - set(self._compressions)
extra = set(self._compressions) - set(self.columns)
- raise DatasetUsageError("compressions don't match columns in %s, missing %s, extra %s" % (self.name, missing or "none", extra or "none",))
+ raise DatasetUsageError(f"compressions don't match columns in {self.name}, missing {missing or 'none'}, extra {extra or 'none'}")
if len(self._lens) != slices:
if self._allow_missing_slices:
for sliceno in range(slices):
self._lens[sliceno] = self._lens.get(sliceno, 0)
else:
- raise DatasetUsageError("Not all slices written, missing %r" % (set(range(slices)) - set(self._lens),))
+ raise DatasetUsageError(f"Not all slices written, missing {set(range(slices)) - set(self._lens)!r}")
args = dict(
columns={k: (v[0].split(':')[-1], v[2]) for k, v in self.columns.items()},
filenames=self._filenames,
@@ -1729,7 +1723,7 @@ def range(self, colname, start=None, stop=None):
"""Filter out only datasets where colname has values in range(start, stop)"""
def predicate(ds):
if colname not in ds.columns:
- raise DatasetUsageError('Dataset %s does not have column %r' % (ds.quoted, colname,))
+ raise DatasetUsageError(f'Dataset {ds.quoted} does not have column {colname!r}')
col = ds.columns[colname]
if col.min is not None:
return (stop is None or col.min < stop) and (start is None or col.max >= start)
@@ -1761,7 +1755,7 @@ class NoDataset(Dataset):
__slots__ = ()
def __new__(cls):
- return unicode.__new__(cls, '')
+ return str.__new__(cls, '')
# functions you shouldn't call on this
append = iterate = iterate_chain = iterate_list = link_to_here = merge = new = None
@@ -1792,7 +1786,7 @@ def range_check_function(bottom, top, none_support=False, index=None):
and/or top to be None. Skips None values if none_support is true."""
if_l = []
d = {}
- v_str = 'v' if index is None else 'v[%d]' % (index,)
+ v_str = 'v' if index is None else f'v[{index}]'
def add_if(op, v):
if if_l:
if_l.append(' and ')
diff --git a/accelerator/dependency.py b/accelerator/dependency.py
index e26a5b9d..4114cf6f 100644
--- a/accelerator/dependency.py
+++ b/accelerator/dependency.py
@@ -18,9 +18,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-
from random import randint
from collections import OrderedDict, defaultdict
from itertools import combinations
diff --git a/accelerator/deptree.py b/accelerator/deptree.py
index 806b8305..87070a6e 100644
--- a/accelerator/deptree.py
+++ b/accelerator/deptree.py
@@ -19,16 +19,13 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-
from traceback import print_exc
from collections import OrderedDict
from datetime import datetime, date, time, timedelta
from pathlib import Path, PosixPath, PurePath, PurePosixPath
import sys
-from accelerator.compat import iteritems, itervalues, first_value, str_types, int_types, num_types, unicode
+from accelerator.compat import iteritems, itervalues, first_value, str_types, int_types, num_types
from accelerator.extras import OptionEnum, OptionEnumValue, _OptionString, OptionDefault, RequiredOption, typing_conv
from accelerator.job import JobWithFile
@@ -77,10 +74,10 @@ def _fix_jobids(self, key):
for jobid_name in method_wants:
if isinstance(jobid_name, str_types):
value = data.get(jobid_name)
- assert value is None or isinstance(value, str), 'Input %s on %s not a string as required' % (jobid_name, method,)
+ assert value is None or isinstance(value, str), f'Input {jobid_name} on {method} not a string as required'
elif isinstance(jobid_name, list):
if len(jobid_name) != 1 or not isinstance(jobid_name[0], str_types):
- raise OptionException('Bad %s item on %s: %s' % (key, method, repr(jobid_name),))
+ raise OptionException(f'Bad {key} item on {method}: {jobid_name!r}')
jobid_name = jobid_name[0]
value = data.get(jobid_name)
if value:
@@ -88,14 +85,14 @@ def _fix_jobids(self, key):
value = [e.strip() for e in value.split(',')]
else:
value = []
- assert isinstance(value, list), 'Input %s on %s not a list or string as required' % (jobid_name, method,)
+ assert isinstance(value, list), f'Input {jobid_name} on {method} not a list or string as required'
else:
- raise OptionException('%s item of unknown type %s on %s: %s' % (key, type(jobid_name), method, repr(jobid_name),))
+ raise OptionException(f'{key} item of unknown type {type(jobid_name)} on {method}: {jobid_name!r}')
res[jobid_name] = value
self.params[key] = res
spill = set(data) - set(res)
if spill:
- raise OptionException('Unknown %s on %s: %s' % (key, method, ', '.join(sorted(spill)),))
+ raise OptionException(f"Unknown {key} on {method}: {', '.join(sorted(spill))}")
def _fix_options(self, fill_in):
method = self.method
@@ -110,13 +107,13 @@ def typefuzz(t):
def convert(default_v, v):
if isinstance(default_v, RequiredOption):
if v is None and not default_v.none_ok:
- raise OptionException('Option %s on method %s requires a non-None value (%r)' % (k, method, default_v.value,))
+ raise OptionException(f'Option {k} on method {method} requires a non-None value ({default_v.value!r})')
default_v = default_v.value
if default_v is None or v is None:
if isinstance(default_v, _OptionString):
- raise OptionException('Option %s on method %s requires a non-empty string value' % (k, method,))
+ raise OptionException(f'Option {k} on method {method} requires a non-empty string value')
if hasattr(default_v, '_valid') and v not in default_v._valid:
- raise OptionException('Option %s on method %s requires a value in %s' % (k, method, default_v._valid,))
+ raise OptionException(f'Option {k} on method {method} requires a value in {default_v._valid}')
if isinstance(default_v, OptionDefault):
v = default_v.default
return v
@@ -148,15 +145,15 @@ def convert(default_v, v):
ok = True
break
if not ok:
- raise OptionException('%r not a permitted value for option %s on method %s (%s)' % (v, k, method, default_v._valid))
+ raise OptionException(f'{v!r} not a permitted value for option {k} on method {method} ({default_v._valid})')
return v or None
if isinstance(default_v, str_types + num_types) and isinstance(v, str_types + num_types):
if isinstance(default_v, _OptionString):
v = str(v)
if not v:
- raise OptionException('Option %s on method %s requires a non-empty string value' % (k, method,))
+ raise OptionException(f'Option {k} on method {method} requires a non-empty string value')
return v
- if isinstance(default_v, unicode) and isinstance(v, bytes):
+ if isinstance(default_v, str) and isinstance(v, bytes):
return v.decode('utf-8')
return type(default_v)(v)
if (isinstance(default_v, type) and isinstance(v, typefuzz(default_v))) or isinstance(v, typefuzz(type(default_v))):
@@ -177,7 +174,7 @@ def convert(default_v, v):
try:
return typing_conv[default_v.__name__](v)
except Exception:
- raise OptionException('Failed to convert option %s %r to %s on method %s' % (k, v, default_v, method,))
+ raise OptionException(f'Failed to convert option {k} {v!r} to {default_v} on method {method}')
if isinstance(v, str_types) and not v:
return type(default_v)()
if isinstance(default_v, JobWithFile) or default_v is JobWithFile:
@@ -185,13 +182,13 @@ def convert(default_v, v):
if default_v is JobWithFile:
default_v = defaults
if not isinstance(v, (list, tuple,)) or not (2 <= len(v) <= 4):
- raise OptionException('Option %s (%r) on method %s is not %s compatible' % (k, v, method, type(default_v)))
+ raise OptionException(f'Option {k} ({v!r}) on method {method} is not {type(default_v)} compatible')
v = tuple(v) + defaults[len(v):] # so all of default_v gets convert()ed.
v = [convert(dv, vv) for dv, vv in zip(default_v, v)]
return JobWithFile(*v)
if type(default_v) != type:
default_v = type(default_v)
- raise OptionException('Failed to convert option %s of %s to %s on method %s' % (k, type(v), default_v, method,))
+ raise OptionException(f'Failed to convert option {k} of {type(v)} to {default_v} on method {method}')
for k, v in iteritems(self.params['options']):
if k in options:
try:
@@ -200,9 +197,9 @@ def convert(default_v, v):
raise
except Exception:
print_exc(file=sys.stderr)
- raise OptionException('Failed to convert option %s on method %s' % (k, method,))
+ raise OptionException(f'Failed to convert option {k} on method {method}')
else:
- raise OptionException('Unknown option %s on method %s' % (k, method,))
+ raise OptionException(f'Unknown option {k} on method {method}')
if fill_in:
missing = set(options) - set(res_options)
missing_required = missing & self.methods.params[method].required
diff --git a/accelerator/dispatch.py b/accelerator/dispatch.py
index 0e812371..7a07c818 100644
--- a/accelerator/dispatch.py
+++ b/accelerator/dispatch.py
@@ -19,13 +19,10 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-
import os
from signal import SIGTERM, SIGKILL
-from accelerator.compat import PY3, monotonic
+from accelerator.compat import monotonic
from accelerator.statmsg import children, statmsg_endwait
from accelerator.build import JobError
@@ -70,10 +67,9 @@ def run(cmd, close_in_child, keep_in_child, no_stdin=True):
devnull = os.open('/dev/null', os.O_RDONLY)
os.dup2(devnull, 0)
os.close(devnull)
- if PY3:
- keep_in_child.update([1, 2])
- for fd in keep_in_child:
- os.set_inheritable(fd, True)
+ keep_in_child.update([1, 2])
+ for fd in keep_in_child:
+ os.set_inheritable(fd, True)
os.execv(cmd[0], cmd)
os._exit()
@@ -85,7 +81,7 @@ def launch(workdir, setup, config, Methods, active_workdirs, slices, concurrency
print_prefix = ''
else:
print_prefix = ' '
- print('%s| %s [%s] |' % (print_prefix, jobid, method,))
+ print(f'{print_prefix}| {jobid} [{method}] |')
args = dict(
workdir=workdir,
slices=slices,
@@ -113,7 +109,7 @@ def launch(workdir, setup, config, Methods, active_workdirs, slices, concurrency
pass
# The dying process won't have sent an end message, so it has
# the endwait time until we SIGKILL it.
- print('%s| %s [%s] failed! (%5.1fs) |' % (print_prefix, jobid, method, monotonic() - starttime))
+ print(f'{print_prefix}| {jobid} [{method}] failed! ({monotonic() - starttime:5.1f}s) |')
# There is a race where stuff on the status socket has not arrived when
# the sending process exits. This is basically benign, but let's give
# it a chance to arrive to cut down on confusing warnings.
@@ -134,5 +130,5 @@ def launch(workdir, setup, config, Methods, active_workdirs, slices, concurrency
pass
if status:
raise JobError(jobid, method, status)
- print('%s| %s [%s] completed. (%5.1fs) |' % (print_prefix, jobid, method, monotonic() - starttime))
+ print(f'{print_prefix}| {jobid} [{method}] completed. ({monotonic() - starttime:5.1f}s) |')
return data
diff --git a/accelerator/dsutil.py b/accelerator/dsutil.py
index ab38d004..0b8ab162 100644
--- a/accelerator/dsutil.py
+++ b/accelerator/dsutil.py
@@ -18,11 +18,8 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-
from accelerator import _dsutil
-from accelerator.compat import str_types, PY3
+from accelerator.compat import str_types
from accelerator.standard_methods._dataset_type import strptime, strptime_i
_convfuncs = {
@@ -68,12 +65,12 @@
def typed_writer(typename):
if typename not in _convfuncs:
- raise ValueError("Unknown writer for type %s" % (typename,))
+ raise ValueError(f"Unknown writer for type {typename}")
return _convfuncs[typename]
def typed_reader(typename):
if typename not in _type2iter:
- raise ValueError("Unknown reader for type %s" % (typename,))
+ raise ValueError(f"Unknown reader for type {typename}")
return _type2iter[typename]
_nodefault = object()
@@ -84,12 +81,8 @@ class WriteJson(object):
min = max = None
def __init__(self, *a, **kw):
default = kw.pop('default', _nodefault)
- if PY3:
- self.fh = _dsutil.WriteUnicode(*a, **kw)
- self.encode = JSONEncoder(ensure_ascii=False, separators=(',', ':')).encode
- else:
- self.fh = _dsutil.WriteBytes(*a, **kw)
- self.encode = JSONEncoder(ensure_ascii=True, separators=(',', ':')).encode
+ self.fh = _dsutil.WriteUnicode(*a, **kw)
+ self.encode = JSONEncoder(ensure_ascii=False, separators=(',', ':')).encode
self.encode = self._wrap_encode(self.encode, default)
def _wrap_encode(self, encode, default):
if default is _nodefault:
@@ -141,10 +134,7 @@ def wrapped_encode(o):
class ReadJson(object):
__slots__ = ('fh', 'decode')
def __init__(self, *a, **kw):
- if PY3:
- self.fh = _dsutil.ReadUnicode(*a, **kw)
- else:
- self.fh = _dsutil.ReadBytes(*a, **kw)
+ self.fh = _dsutil.ReadUnicode(*a, **kw)
self.decode = JSONDecoder().decode
def __next__(self):
return self.decode(next(self.fh))
@@ -167,7 +157,6 @@ class WritePickle(object):
__slots__ = ('fh',)
min = max = None
def __init__(self, *a, **kw):
- assert PY3, "Pickle columns require python 3, sorry"
assert 'default' not in kw, "default not supported for Pickle, sorry"
self.fh = _dsutil.WriteBytes(*a, **kw)
def write(self, o):
@@ -189,7 +178,6 @@ def __exit__(self, type, value, traceback):
class ReadPickle(object):
__slots__ = ('fh',)
def __init__(self, *a, **kw):
- assert PY3, "Pickle columns require python 3, sorry"
self.fh = _dsutil.ReadBytes(*a, **kw)
def __next__(self):
return pickle_loads(next(self.fh))
@@ -229,7 +217,7 @@ def _sanity_check_float_hashing():
def check(typ, msg, want, *a):
for ix, v in enumerate(a):
if v != want:
- raise _SanityError("%s did not hash %s value correctly. (%d)" % (typ, msg, ix,))
+ raise _SanityError(f"{typ} did not hash {msg} value correctly. ({ix})")
# Test that the float types (including number) hash floats the same,
# and that float32 rounds as expected.
diff --git a/accelerator/error.py b/accelerator/error.py
index 7fd5abd1..086d1bdf 100644
--- a/accelerator/error.py
+++ b/accelerator/error.py
@@ -18,10 +18,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
class AcceleratorError(Exception):
"""Base class for all accelerator exception types"""
__slots__ = ()
@@ -73,15 +69,15 @@ class BuildError(AcceleratorError):
class JobError(BuildError):
def __init__(self, job, method, status):
- AcceleratorError.__init__(self, "Failed to build %s (%s)" % (job, method,))
+ AcceleratorError.__init__(self, f"Failed to build {job} ({method})")
self.job = job
self.method = method
self.status = status
def format_msg(self):
- res = ["%s (%s):" % (self.job, self.method,)]
+ res = [f"{self.job} ({self.method}):"]
for component, msg in self.status.items():
- res.append(" %s:" % (component,))
+ res.append(f" {component}:")
res.append(" %s" % (msg.replace("\n", "\n "),))
return "\n".join(res)
diff --git a/accelerator/examples/a_example_returninprepanasyn.py b/accelerator/examples/a_example_returninprepanasyn.py
index be1dff66..d5d75d80 100644
--- a/accelerator/examples/a_example_returninprepanasyn.py
+++ b/accelerator/examples/a_example_returninprepanasyn.py
@@ -6,7 +6,7 @@ def prepare():
def analysis(sliceno, prepare_res):
- return 'this is analysis %s with prepare_res=%s' % (str(sliceno), prepare_res)
+ return f'this is analysis {sliceno!s} with prepare_res={prepare_res}'
def synthesis(analysis_res, prepare_res):
diff --git a/accelerator/examples/a_example_writeslicedfile.py b/accelerator/examples/a_example_writeslicedfile.py
index 23a20b1f..8a936262 100644
--- a/accelerator/examples/a_example_writeslicedfile.py
+++ b/accelerator/examples/a_example_writeslicedfile.py
@@ -4,16 +4,16 @@
def analysis(sliceno, job):
# save one file per analysis process...
filename = 'myfile1'
- data = 'This is job %s analysis slice %d.' % (job, sliceno,)
+ data = f'This is job {job} analysis slice {sliceno}.'
job.save(data, filename, sliceno=sliceno)
def synthesis(job):
# ...and one file in the synthesis process...
filename = 'myfile2'
- data = 'this is job %s synthesis' % (job,)
+ data = f'this is job {job} synthesis'
job.save(data, filename)
# ...and let's return some data too.
- returndata = 'this is job %s return value' % (job,)
+ returndata = f'this is job {job} return value'
return returndata
diff --git a/accelerator/examples/build_dsexample-chain.py b/accelerator/examples/build_dsexample-chain.py
index c965d098..645b21f9 100644
--- a/accelerator/examples/build_dsexample-chain.py
+++ b/accelerator/examples/build_dsexample-chain.py
@@ -26,9 +26,9 @@ def main(urd):
prt()
prt('To go back in chain and investigate datasets, try')
- prt.command('ax ds %s' % (imp,))
- prt.command('ax ds %s~' % (imp,))
- prt.command('ax ds %s~~' % (imp,))
+ prt.command(f'ax ds {imp}')
+ prt.command(f'ax ds {imp}~')
+ prt.command(f'ax ds {imp}~~')
prt('Note that ~~ can also be written ~2 etc.')
prt()
diff --git a/accelerator/examples/build_dsexample-create.py b/accelerator/examples/build_dsexample-create.py
index dcd58e7b..1a8d6664 100644
--- a/accelerator/examples/build_dsexample-create.py
+++ b/accelerator/examples/build_dsexample-create.py
@@ -19,4 +19,4 @@ def main(urd):
prt()
prt('For convenience, the jobid can be used as a reference to the default')
- prt('dataset in a job. The full name is "%s/default".' % (job,))
+ prt(f'dataset in a job. The full name is "{job}/default".')
diff --git a/accelerator/examples/build_dsexample-many_ds.py b/accelerator/examples/build_dsexample-many_ds.py
index 861c9196..487335e8 100644
--- a/accelerator/examples/build_dsexample-many_ds.py
+++ b/accelerator/examples/build_dsexample-many_ds.py
@@ -20,4 +20,4 @@ def main(urd):
prt()
prt('Print contents of a specific dataset like this')
- prt.command('ax cat -H %s/third' % (job,))
+ prt.command(f'ax cat -H {job}/third')
diff --git a/accelerator/examples/build_example-depend_extra.py b/accelerator/examples/build_example-depend_extra.py
index 98bcb46d..97642623 100644
--- a/accelerator/examples/build_example-depend_extra.py
+++ b/accelerator/examples/build_example-depend_extra.py
@@ -16,4 +16,4 @@ def main(urd):
prt()
prt('source code is here:')
- prt.plain(join(dirname(__file__), 'a_%s.py' % (job.method,)))
+ prt.plain(join(dirname(__file__), f'a_{job.method}.py'))
diff --git a/accelerator/examples/build_example-equiv_hashes.py b/accelerator/examples/build_example-equiv_hashes.py
index 744d8be1..7047fdee 100644
--- a/accelerator/examples/build_example-equiv_hashes.py
+++ b/accelerator/examples/build_example-equiv_hashes.py
@@ -15,7 +15,7 @@ def main(urd):
thehash = job.params.hash
prt()
prt('Take the hash from the built job:')
- prt.output('"%s"' % (thehash,))
+ prt.output(f'"{thehash}"')
prt('and add it to the method like this')
prt.output('"equivalent_hashes = {\'whatever\': (\'%s\',)}"' % (thehash,))
@@ -32,4 +32,4 @@ def main(urd):
prt('to the old one.')
prt()
prt('Method source file is here')
- prt.plain(join(dirname(__file__), 'a_%s.py' % (job.method,)))
+ prt.plain(join(dirname(__file__), f'a_{job.method}.py'))
diff --git a/accelerator/examples/build_example-files.py b/accelerator/examples/build_example-files.py
index 35b510aa..fa3776f4 100644
--- a/accelerator/examples/build_example-files.py
+++ b/accelerator/examples/build_example-files.py
@@ -10,7 +10,7 @@ def main(urd):
job1 = urd.build('example_writeslicedfile')
prt()
- prt('These are the files created by / located in job %s.' % (job1,))
+ prt(f'These are the files created by / located in job {job1}.')
prt.output(job1.files())
prt()
@@ -23,13 +23,13 @@ def main(urd):
)
prt()
- prt('Read and print stored stdout from %s synthesis' % (job2,))
+ prt(f'Read and print stored stdout from {job2} synthesis')
prt.output(job2.output('synthesis'))
prt()
- prt('Read and print stored stdout from %s everything' % (job2,))
+ prt(f'Read and print stored stdout from {job2} everything')
prt.output(job2.output())
prt()
- prt('Read and print stored stdout from %s analysis process 2' % (job2,))
+ prt(f'Read and print stored stdout from {job2} analysis process 2')
prt.output(job2.output(2))
prt()
@@ -40,4 +40,4 @@ def main(urd):
prt.output(pickle.load(fh))
prt('To see all files stored in a job, try')
- prt.command('ax job %s' % (job1,))
+ prt.command(f'ax job {job1}')
diff --git a/accelerator/examples/build_example-options.py b/accelerator/examples/build_example-options.py
index cdd4b5b7..4419b470 100644
--- a/accelerator/examples/build_example-options.py
+++ b/accelerator/examples/build_example-options.py
@@ -22,11 +22,11 @@ def main(urd):
prt('The job will print its options to stdout.')
prt()
prt('See the options as the job sees them by running')
- prt.command('ax job -O %s' % (job,))
+ prt.command(f'ax job -O {job}')
prt('The "ax job" command will also show the job\'s options')
- prt.command('ax job %s' % (job,))
+ prt.command(f'ax job {job}')
prt('See everything using')
- prt.command('ax job -o %s' % (job,))
+ prt.command(f'ax job -o {job}')
prt()
prt.header('Short and long version (see code)')
diff --git a/accelerator/examples/build_tutorial01.py b/accelerator/examples/build_tutorial01.py
index 5d8e0390..8354f28a 100644
--- a/accelerator/examples/build_tutorial01.py
+++ b/accelerator/examples/build_tutorial01.py
@@ -38,9 +38,9 @@ def main(urd):
reference to the job that created the dataset as a reference
to the default dataset inside the job. I.e.
''')
- prt.command('ax ds %s' % (imp,))
+ prt.command(f'ax ds {imp}')
prt('is equivalent to')
- prt.command('ax ds %s/default' % (imp,))
+ prt.command(f'ax ds {imp}/default')
prt('which is the formally correct way to refer to the dataset.')
prt()
diff --git a/accelerator/examples/build_tutorial02.py b/accelerator/examples/build_tutorial02.py
index e5b88f2f..b8c550a2 100644
--- a/accelerator/examples/build_tutorial02.py
+++ b/accelerator/examples/build_tutorial02.py
@@ -37,8 +37,8 @@ def main(urd):
prt()
with prt.header('VISUALISING A JOB\'S DEPENDENCIES'):
prt('View the job\'s metadata using')
- prt.command('ax job %s' % (imp,))
- prt('We can see that the dataset "%s" is input to this job.' % (imp.params.datasets.source,))
+ prt.command(f'ax job {imp}')
+ prt(f'We can see that the dataset "{imp.params.datasets.source}" is input to this job.')
prt()
with prt.header('REFERENCES TO ALL JOBS ARE STORED IN THE "urd.joblist" OBJECT:'):
@@ -54,7 +54,7 @@ def main(urd):
Take a look at the dataset created by dataset_hashpart:
''')
- prt.command('ax ds %s' % (imp,))
+ prt.command(f'ax ds {imp}')
prt('''
The asterisk on the row corresponding to the "String" column
indicates that the dataset is hash partitioned based the values
@@ -63,4 +63,4 @@ def main(urd):
We can also see how many rows there are in each slice by typing
''')
- prt.command('ax ds -s %s' % (imp,))
+ prt.command(f'ax ds -s {imp}')
diff --git a/accelerator/examples/build_tutorial03.py b/accelerator/examples/build_tutorial03.py
index ee4daef6..10886569 100644
--- a/accelerator/examples/build_tutorial03.py
+++ b/accelerator/examples/build_tutorial03.py
@@ -64,16 +64,16 @@ def main(urd):
with prt.header('INSPECTING DATASET CHAINS'):
prt(' The flags -s, -S, and -c to "ax ds" are useful when looking')
prt(' at chained datasets')
- prt.command('ax ds -s %s' % (imp,))
- prt.command('ax ds -S %s' % (imp,))
- prt.command('ax ds -c %s' % (imp,))
+ prt.command(f'ax ds -s {imp}')
+ prt.command(f'ax ds -S {imp}')
+ prt.command(f'ax ds -c {imp}')
prt('As always, see info and more options using')
prt.command('ax ds --help')
prt()
with prt.header('PRINTING A DATASET CHAIN'):
prt('This is a small example, so we can print all data using')
- prt.command('ax cat -c %s' % (imp,))
+ prt.command(f'ax cat -c {imp}')
prt()
with prt.header('GREPPING'):
@@ -82,4 +82,4 @@ def main(urd):
"String" column, but show "Date", "Float", and "Int"
columns only.
''')
- prt.command('ax grep bb -c -g String %s Date Float Int' % (imp,))
+ prt.command(f'ax grep bb -c -g String {imp} Date Float Int')
diff --git a/accelerator/examples/build_tutorial04.py b/accelerator/examples/build_tutorial04.py
index e856514f..82f88266 100644
--- a/accelerator/examples/build_tutorial04.py
+++ b/accelerator/examples/build_tutorial04.py
@@ -71,4 +71,4 @@ def main(urd):
prt()
prt('Try')
prt.command('ax job -O', job)
- prt('to print this output from the %s job to the terminal.' % (job.method,))
+ prt(f'to print this output from the {job.method} job to the terminal.')
diff --git a/accelerator/examples/build_tutorial05.py b/accelerator/examples/build_tutorial05.py
index 6bac03c3..ab0511aa 100644
--- a/accelerator/examples/build_tutorial05.py
+++ b/accelerator/examples/build_tutorial05.py
@@ -53,11 +53,11 @@ def main(urd):
urd.finish(key)
prt()
- prt('''
- Now, references to everything that has been built, including
- job dependencies, is stored in the Urd server using the.
- key "/%s".
- ''' % (key,))
+ prt(f'''
+\t\tNow, references to everything that has been built, including
+\t\tjob dependencies, is stored in the Urd server using the.
+\t\tkey "/{key}".
+\t''')
prt()
with prt.header('View all Urd lists.'):
@@ -70,10 +70,10 @@ def main(urd):
prt()
with prt.header('Inspecting all sessions in a list.'):
prt('All sessions are timestamped. To see all timestamps')
- prt('and captions in "%s" since timestamp zero, do' % (key,))
- prt.command('ax urd %s/since/0' % (key,))
+ prt(f'and captions in "{key}" since timestamp zero, do')
+ prt.command(f'ax urd {key}/since/0')
prt('or equivalently')
- prt.command('ax urd %s/' % (key,))
+ prt.command(f'ax urd {key}/')
prt('the output from this command looks something like this (try it)')
for ts in urd.since(key, 0):
prt.output(ts, urd.peek(key, ts).caption)
@@ -81,8 +81,8 @@ def main(urd):
prt()
with prt.header('Inspecting individual Urd items.'):
prt('We can look at an individual Urd-item like this')
- prt.command('ax urd %s/3' % (key,))
- prt('which corresponds to the list in key "%s" at timestamp 3.' % (key,))
+ prt.command(f'ax urd {key}/3')
+ prt(f'which corresponds to the list in key "{key}" at timestamp 3.')
prt('(Timestamps can be dates, datetimes, integers, or tuples')
prt('of date/datetimes and integers.)')
diff --git a/accelerator/examples/build_urdexample-basic.py b/accelerator/examples/build_urdexample-basic.py
index 80846411..92fbda3a 100644
--- a/accelerator/examples/build_urdexample-basic.py
+++ b/accelerator/examples/build_urdexample-basic.py
@@ -14,7 +14,7 @@ def main(urd):
prt()
prt('Create and Urd item containing a joblist of one job.')
- prt('The name of the Urd list is "%s".' % (listname,))
+ prt(f'The name of the Urd list is "{listname}".')
urd.begin(listname, timestamp, caption='you may assign a caption here')
urd.build('example_returnhelloworld')
urd.finish(listname)
@@ -30,34 +30,34 @@ def main(urd):
prt('# will do the same thing on the command line.')
prt()
- prt('Here\'s a list of all entries in the list "%s"' % (listname,))
+ prt(f'Here\'s a list of all entries in the list "{listname}"')
for ts in urd.since(listname, 0):
prt.output(ts)
prt()
prt('The command')
- prt.command('ax urd %s/since/0' % (listname,))
+ prt.command(f'ax urd {listname}/since/0')
prt('or')
- prt.command('ax urd %s/' % (listname,))
+ prt.command(f'ax urd {listname}/')
prt('will do the same thing on the command line.')
prt()
prt('To see a specific entry, try')
- prt.command('ax urd %s/%s' % (listname, timestamp))
+ prt.command(f'ax urd {listname}/{timestamp}')
prt()
prt('To see information about the a specific job in that Urd session')
- prt.command('ax job :%s/%s:example1' % (listname, timestamp))
+ prt.command(f'ax job :{listname}/{timestamp}:example1')
prt()
prt('To see information about the last job in that Urd session')
- prt.command('ax job :%s/%s:-1' % (listname, timestamp))
- prt.command('ax job :%s/%s:' % (listname, timestamp))
+ prt.command(f'ax job :{listname}/{timestamp}:-1')
+ prt.command(f'ax job :{listname}/{timestamp}:')
prt('Note that the commands defaults to the _last_ item')
prt()
prt('To see information about the last job in the last Urd session')
- prt.command('ax job :%s:' % (listname,))
+ prt.command(f'ax job :{listname}:')
prt('Note that the commands defaults to the _last_ item')
prt()
diff --git a/accelerator/examples/build_urdexample-many_items.py b/accelerator/examples/build_urdexample-many_items.py
index 67aed736..f75f2c48 100644
--- a/accelerator/examples/build_urdexample-many_items.py
+++ b/accelerator/examples/build_urdexample-many_items.py
@@ -14,7 +14,7 @@ def main(urd):
urd.truncate(listname, 0)
prt('Start by creating ten chained jobs. Each job is associated')
- prt('with a unique Urd-item in the "%s" Urd list.' % (listname,))
+ prt(f'with a unique Urd-item in the "{listname}" Urd list.')
starttime = date(2021, 8, 1)
for days in range(10):
timestamp = starttime + timedelta(days=days)
@@ -24,11 +24,11 @@ def main(urd):
urd.finish(listname)
prt()
- prt('Here\'s a list of all entries in the list "%s"' % (listname,))
+ prt(f'Here\'s a list of all entries in the list "{listname}"')
for ts in urd.since(listname, 0):
- prt.output(ts, '"%s"' % (urd.peek(listname, ts).caption,))
+ prt.output(ts, f'"{urd.peek(listname, ts).caption}"')
prt('This is available in the shell using')
- prt.command('ax urd %s/' % (listname,))
+ prt.command(f'ax urd {listname}/')
prt()
prt('Here\'s the session for timestamp 2021-08-06')
@@ -39,4 +39,4 @@ def main(urd):
prt()
prt('To print this in a more human friendly format on the command line, try')
- prt.command(' ax urd %s/2021-08-06' % (listname,))
+ prt.command(f' ax urd {listname}/2021-08-06')
diff --git a/accelerator/extras.py b/accelerator/extras.py
index 6fd96121..7cfcbf4a 100644
--- a/accelerator/extras.py
+++ b/accelerator/extras.py
@@ -20,9 +20,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-
import os
import datetime
import json
@@ -32,8 +29,8 @@
from functools import partial
import sys
-from accelerator.compat import PY2, PY3, pickle, izip, iteritems, first_value
-from accelerator.compat import num_types, uni, unicode, str_types
+from accelerator.compat import pickle, izip, iteritems, first_value
+from accelerator.compat import num_types, str_types
from accelerator.error import AcceleratorError
from accelerator.job import Job, JobWithFile
@@ -43,12 +40,12 @@ def _fn(filename, jobid, sliceno):
if isinstance(filename, pathlib.Path):
filename = str(filename)
if filename.startswith('/'):
- assert not jobid, "Don't specify full path (%r) and jobid (%s)." % (filename, jobid,)
- assert not sliceno, "Don't specify full path (%r) and sliceno." % (filename,)
+ assert not jobid, f"Don't specify full path ({filename!r}) and jobid ({jobid})."
+ assert not sliceno, f"Don't specify full path ({filename!r}) and sliceno."
elif jobid:
filename = Job(jobid).filename(filename, sliceno)
elif sliceno is not None:
- filename = '%s.%d' % (filename, sliceno,)
+ filename = f'{filename}.{sliceno}'
return filename
def _type_list_or(v, t, falseval, listtype):
@@ -189,7 +186,7 @@ def job_post(jobid):
d.files = sorted(fn[len(prefix):] if fn.startswith(prefix) else fn for fn in d.files)
version = 1
if version != 1:
- raise AcceleratorError("Don't know how to load post.json version %d (in %s)" % (d.version, jobid,))
+ raise AcceleratorError(f"Don't know how to load post.json version {d.version} (in {jobid})")
return d
def _pickle_save(variable, filename, temp, _hidden):
@@ -210,12 +207,9 @@ def pickle_save(variable, filename='result.pickle', sliceno=None, temp=None, bac
# default is 'ascii', which is pretty terrible too.)
def pickle_load(filename='result.pickle', jobid=None, sliceno=None, encoding='bytes'):
filename = _fn(filename, jobid, sliceno)
- with status('Loading ' + filename):
+ with status(f'Loading {filename}'):
with open(filename, 'rb') as fh:
- if PY3:
- return pickle.load(fh, encoding=encoding)
- else:
- return pickle.load(fh)
+ return pickle.load(fh, encoding=encoding)
def json_encode(variable, sort_keys=True, as_str=False):
@@ -238,13 +232,11 @@ def typefix(e):
return dict_type((typefix(k), typefix(v)) for k, v in iteritems(e))
elif isinstance(e, (list, tuple, set,)):
return [typefix(v) for v in e]
- elif PY2 and isinstance(e, bytes):
- return uni(e)
else:
return e
variable = typefix(variable)
res = json.dumps(variable, indent=4, sort_keys=sort_keys)
- if PY3 and not as_str:
+ if not as_str:
res = res.encode('ascii')
return res
@@ -262,7 +254,7 @@ def json_save(variable, filename='result.json', sliceno=None, sort_keys=True, _e
return _SavedFile(filename, sliceno, json_load)
def _unicode_as_utf8bytes(obj):
- if isinstance(obj, unicode):
+ if isinstance(obj, str):
return obj.encode('utf-8')
elif isinstance(obj, dict):
return DotDict((_unicode_as_utf8bytes(k), _unicode_as_utf8bytes(v)) for k, v in iteritems(obj))
@@ -271,30 +263,23 @@ def _unicode_as_utf8bytes(obj):
else:
return obj
-def json_decode(s, unicode_as_utf8bytes=PY2):
+def json_decode(s, unicode_as_utf8bytes=False):
if unicode_as_utf8bytes:
return _unicode_as_utf8bytes(json.loads(s, object_pairs_hook=DotDict))
else:
return json.loads(s, object_pairs_hook=DotDict)
-def json_load(filename='result.json', jobid=None, sliceno=None, unicode_as_utf8bytes=PY2):
+def json_load(filename='result.json', jobid=None, sliceno=None, unicode_as_utf8bytes=False):
filename = _fn(filename, jobid, sliceno)
- if PY3:
- with open(filename, 'r', encoding='utf-8') as fh:
- data = fh.read()
- else:
- with open(filename, 'rb') as fh:
- data = fh.read()
+ with open(filename, 'r', encoding='utf-8') as fh:
+ data = fh.read()
return json_decode(data, unicode_as_utf8bytes)
def quote(s):
"""Quote s unless it looks fine without"""
- s = unicode(s)
+ s = str(s)
r = repr(s)
- if PY2:
- # remove leading u
- r = r[1:]
if s and len(s) + 2 == len(r) and not any(c.isspace() for c in s):
return s
else:
@@ -308,7 +293,7 @@ def debug_print_options(options, title=''):
print('-' * 53)
max_k = max(len(str(k)) for k in options)
for key, val in sorted(options.items()):
- print("%s = %r" % (str(key).ljust(max_k), val))
+ print(f"{str(key).ljust(max_k)} = {val!r}")
print('-' * 53)
@@ -339,15 +324,14 @@ def __init__(self, filename, temp=None, _hidden=False):
# Normalise filename so it always contains the job path (unless already absolute)
filename = job.filename(filename)
self.filename = filename
- self.tmp_filename = '%s.%dtmp' % (filename, os.getpid(),)
+ self.tmp_filename = f'{filename}.{os.getpid()}tmp'
self.temp = temp
self._hidden = _hidden
def __enter__(self):
- self._status = status('Saving ' + self.filename)
+ self._status = status(f'Saving {self.filename}')
self._status.__enter__()
- # stupid python3 feels that w and x are exclusive, while python2 requires both.
- fh = getattr(self, '_open', open)(self.tmp_filename, 'xb' if PY3 else 'wbx')
+ fh = getattr(self, '_open', open)(self.tmp_filename, 'xb')
self.close = fh.close
return fh
def __exit__(self, e_type, e_value, e_tb):
@@ -381,7 +365,7 @@ def __iter__(self):
return self
def _loader(self, ix, slices):
for sliceno in slices:
- yield pickle_load("Analysis.%d." % (ix,), sliceno=sliceno)
+ yield pickle_load(f"Analysis.{ix}.", sliceno=sliceno)
def __next__(self):
if self._is_tupled:
return next(self._tupled)
@@ -457,13 +441,13 @@ def _merge_auto_single(self, it, ix):
to_check = data
while hasattr(to_check, "values"):
if not to_check:
- raise self._exc("Empty value at depth %d (index %d)" % (depth, ix,))
+ raise self._exc(f"Empty value at depth {depth} (index {ix})")
to_check = first_value(to_check)
depth += 1
if hasattr(to_check, "update"): # like a set
depth += 1
if not depth:
- raise self._exc("Top level has no .values (index %d)" % (ix,))
+ raise self._exc(f"Top level has no .values (index {ix})")
def upd(aggregate, part, level):
if level == depth:
aggregate.update(part)
@@ -505,11 +489,6 @@ def __getattr__(self, name):
raise AttributeError(name)
def __setattr__(self, name, value):
- # if using the python implementation of OrderedDict (as python2 does)
- # this is needed. don't worry about __slots__, it won't apply in that
- # case, and __getattr__ is not needed as it falls through automatically.
- if name.startswith('_OrderedDict__'):
- return OrderedDict.__setattr__(self, name, value)
if name[0] == "_":
raise AttributeError(name)
self[name] = value
@@ -544,12 +523,11 @@ def __radd__(self, other):
return self.__class__(list.__add__(other, self))
def __repr__(self):
- return '%s(%s)' % (self.__class__.__name__, list.__repr__(self))
+ return f'{self.__class__.__name__}({list.__repr__(self)})'
class OptionEnumValue(str):
- if PY3: # python2 doesn't support slots on str subclasses
- __slots__ = ('_valid', '_prefixes')
+ __slots__ = ('_valid', '_prefixes')
@staticmethod
def _mktype(name, valid, prefixes):
@@ -600,8 +578,6 @@ def __new__(cls, values, none_ok=False):
if isinstance(values, str_types):
values = values.replace(',', ' ').split()
values = list(values)
- if PY2:
- values = [v.encode('utf-8') if isinstance(v, unicode) else v for v in values]
valid = set(values)
prefixes = []
for v in values:
@@ -643,7 +619,7 @@ def __call__(self, example):
return _OptionString(example)
def __repr__(self):
if self:
- return 'OptionString(%r)' % (str(self),)
+ return f'OptionString({str(self)!r})'
else:
return 'OptionString'
OptionString = _OptionString('')
diff --git a/accelerator/graph.py b/accelerator/graph.py
index 5a371754..aabc5ec9 100644
--- a/accelerator/graph.py
+++ b/accelerator/graph.py
@@ -23,7 +23,7 @@
from datetime import datetime
from accelerator import JobWithFile, Job
from accelerator.dataset import Dataset
-from accelerator.compat import unicode, FileNotFoundError
+from accelerator.compat import FileNotFoundError
MAXANGLE = 45 * pi / 180
@@ -74,7 +74,7 @@ def dsdeps(ds):
return res
-class WrapperNode(unicode):
+class WrapperNode(str):
def __init__(self, payload):
self.payload = payload
self.level = 0
@@ -249,7 +249,7 @@ def create_graph(inputitem, urdinfo=()):
else:
# dataset
n.columns = sorted((colname, dscol.type) for colname, dscol in n.payload.columns.items())
- n.lines = "%d x % s" % (len(n.payload.columns), '{:,}'.format(sum(n.payload.lines)).replace(',', '_'))
+ n.lines = "%d x % s" % (len(n.payload.columns), f'{sum(n.payload.lines):,}'.replace(',', '_'))
n.ds = n.payload
graph.populate_with_neighbours()
return graph
@@ -314,7 +314,7 @@ def graph(inp, gtype):
jobid2urddep = defaultdict(list)
for key, urditem in inp.deps.items():
for _, jid in urditem.joblist:
- jobid2urddep[jid].append("%s/%s" % (key, urditem.timestamp))
+ jobid2urddep[jid].append(f"{key}/{urditem.timestamp}")
jobid2urddep = {key: sorted(val) for key, val in jobid2urddep.items()}
jlist = inp.joblist
inp = tuple(Job(jid) for _, jid in jlist)
diff --git a/accelerator/iowrapper.py b/accelerator/iowrapper.py
index c8fb012a..bb428576 100644
--- a/accelerator/iowrapper.py
+++ b/accelerator/iowrapper.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
import os
from contextlib import contextmanager
from multiprocessing import Process
@@ -130,7 +126,7 @@ def reader(fd2pid, names, masters, slaves, process_name, basedir, is_main, is_bu
outputs = dict.fromkeys(masters, b'')
if len(fd2pid) == 2:
status_blacklist = set(fd2pid.values())
- assert len(status_blacklist) == 1, "fd2pid should only map to 1 value initially: %r" % (fd2pid,)
+ assert len(status_blacklist) == 1, f"fd2pid should only map to 1 value initially: {fd2pid!r}"
else:
status_blacklist = ()
assert len(fd2pid) == 1, "fd2pid should have 1 or 2 elements initially"
diff --git a/accelerator/job.py b/accelerator/job.py
index d02c6621..a24996bb 100644
--- a/accelerator/job.py
+++ b/accelerator/job.py
@@ -27,7 +27,7 @@
from functools import wraps
from pathlib import Path
-from accelerator.compat import unicode, PY2, PY3, open, iteritems, FileNotFoundError
+from accelerator.compat import iteritems, FileNotFoundError
from accelerator.error import NoSuchJobError, NoSuchWorkdirError, NoSuchDatasetError, AcceleratorError
@@ -44,9 +44,9 @@ def dirnamematcher(name):
def _assert_is_normrelpath(path, dirtype):
norm = os.path.normpath(path)
if (norm != path and norm + '/' != path) or norm.startswith('/'):
- raise AcceleratorError('%r is not a normalised relative path' % (path,))
+ raise AcceleratorError(f'{path!r} is not a normalised relative path')
if norm == '..' or norm.startswith('../'):
- raise AcceleratorError('%r is above the %s dir' % (path, dirtype))
+ raise AcceleratorError(f'{path!r} is above the {dirtype} dir')
def _cachedprop(meth):
@@ -61,7 +61,7 @@ def wrapper(self):
_cache = {}
_nodefault = object()
-class Job(unicode):
+class Job(str):
"""
A string that is a jobid, but also has some extra properties:
.method The job method (can be the "name" when from build or urd).
@@ -82,7 +82,7 @@ class Job(unicode):
.link_result to put a link in result_directory that points to a file in this job.
.link_to_here to expose a subjob result in its parent.
- Decays to a (unicode) string when pickled.
+ Decays to a string when pickled.
"""
__slots__ = ('workdir', 'number', '_cache')
@@ -91,12 +91,12 @@ def __new__(cls, jobid, method=None):
k = (jobid, method)
if k in _cache:
return _cache[k]
- obj = unicode.__new__(cls, jobid)
+ obj = str.__new__(cls, jobid)
try:
obj.workdir, tmp = jobid.rsplit('-', 1)
obj.number = int(tmp)
except ValueError:
- raise NoSuchJobError('Not a valid jobid: "%s".' % (jobid,))
+ raise NoSuchJobError(f'Not a valid jobid: "{jobid}".')
obj._cache = {}
if method:
obj._cache['method'] = method
@@ -105,7 +105,7 @@ def __new__(cls, jobid, method=None):
@classmethod
def _create(cls, name, number):
- return Job('%s-%d' % (name, number,))
+ return Job(f'{name}-{number}')
@_cachedprop
def method(self):
@@ -134,14 +134,14 @@ def build_job(self):
@property
def path(self):
if self.workdir not in WORKDIRS:
- raise NoSuchWorkdirError('Not a valid workdir: "%s"' % (self.workdir,))
+ raise NoSuchWorkdirError(f'Not a valid workdir: "{self.workdir}"')
return os.path.join(WORKDIRS[self.workdir], self)
def filename(self, filename, sliceno=None):
if isinstance(filename, Path):
filename = str(filename)
if sliceno is not None:
- filename = '%s.%d' % (filename, sliceno,)
+ filename = f'{filename}.{sliceno}'
return os.path.join(self.path, filename)
def open(self, filename, mode='r', sliceno=None, encoding=None, errors=None):
@@ -196,7 +196,7 @@ def load(self, filename='result.pickle', sliceno=None, encoding='bytes', default
raise
return default
- def json_load(self, filename='result.json', sliceno=None, unicode_as_utf8bytes=PY2, default=_nodefault):
+ def json_load(self, filename='result.json', sliceno=None, unicode_as_utf8bytes=False, default=_nodefault):
from accelerator.extras import json_load
try:
return json_load(self.filename(filename, sliceno), unicode_as_utf8bytes=unicode_as_utf8bytes)
@@ -223,7 +223,7 @@ def output(self, what=None):
if isinstance(what, int):
fns = [what]
else:
- assert what in (None, 'prepare', 'analysis', 'synthesis'), 'Unknown output %r' % (what,)
+ assert what in (None, 'prepare', 'analysis', 'synthesis'), f'Unknown output {what!r}'
if what in (None, 'analysis'):
fns = list(range(self.params.slices))
if what is None:
@@ -262,14 +262,14 @@ def link_result(self, filename='result.pickle', linkname=None, header=None, desc
else:
linkname += os.path.basename(filename)
source_fn = os.path.join(self.path, filename)
- assert os.path.exists(source_fn), "Filename \"%s\" does not exist in jobdir \"%s\"!" % (filename, self.path)
+ assert os.path.exists(source_fn), f"Filename \"{filename}\" does not exist in jobdir \"{self.path}\"!"
result_directory = cfg['result_directory']
dest_fn = result_directory
for part in linkname.split('/'):
if not os.path.exists(dest_fn):
os.mkdir(dest_fn)
elif dest_fn != result_directory and os.path.islink(dest_fn):
- raise AcceleratorError("Refusing to create link %r: %r is a symlink" % (linkname, dest_fn))
+ raise AcceleratorError(f"Refusing to create link {linkname!r}: {dest_fn!r} is a symlink")
dest_fn = os.path.join(dest_fn, part)
try:
os.remove(dest_fn + '_')
@@ -318,7 +318,7 @@ def chain(self, length=-1, reverse=False, stop_job=None):
# Look like a string after pickling
def __reduce__(self):
- return unicode, (unicode(self),)
+ return str, (str(self),)
class CurrentJob(Job):
@@ -360,7 +360,7 @@ def open(self, filename, mode='r', sliceno=None, encoding=None, errors=None, tem
return Job.open(self, filename, mode, sliceno, encoding, errors)
if 'b' not in mode and encoding is None:
encoding = 'utf-8'
- if PY3 and 'x' not in mode:
+ if 'x' not in mode:
mode = mode.replace('w', 'x')
def _open(fn, _mode):
# ignore the passed mode, use the one we have
@@ -380,11 +380,11 @@ def register_file(self, filename):
from accelerator.extras import saved_files
saved_files[filename] = 0
- def register_files(self, pattern='**/*' if PY3 else '*'):
+ def register_files(self, pattern='**/*'):
"""Bulk register files matching a pattern.
Tries to exclude internal files automatically.
Does not register temp-files.
- The default pattern registers everything (recursively, unless python 2).
+ The default pattern registers everything, recursively.
Returns which files were registered.
"""
from accelerator.extras import saved_files
@@ -394,11 +394,7 @@ def register_files(self, pattern='**/*' if PY3 else '*'):
assert not pattern.startswith('../')
forbidden = ('setup.json', 'post.json', 'method.tar.gz', 'link_result.pickle')
res = set()
- if PY3:
- files = iglob(pattern, recursive=True)
- else:
- # No recursive support on python 2.
- files = iglob(pattern)
+ files = iglob(pattern, recursive=True)
for fn in files:
if (
fn in forbidden or
@@ -444,7 +440,7 @@ class NoJob(Job):
number = version = -1
def __new__(cls):
- return unicode.__new__(cls, '')
+ return str.__new__(cls, '')
def dataset(self, name='default'):
raise NoSuchDatasetError('NoJob has no datasets')
@@ -464,7 +460,7 @@ def load(self, filename=None, sliceno=None, encoding='bytes', default=_nodefault
raise NoSuchJobError('Can not load named / sliced file on ')
return None
- def json_load(self, filename=None, sliceno=None, unicode_as_utf8bytes=PY2, default=_nodefault):
+ def json_load(self, filename=None, sliceno=None, unicode_as_utf8bytes=False, default=_nodefault):
return self.load(filename, sliceno, default=default)
@property # so it can return the same instance as all other NoJob things
@@ -500,7 +496,7 @@ def load(self, sliceno=None, encoding='bytes', default=_nodefault):
raise
return default
- def json_load(self, sliceno=None, unicode_as_utf8bytes=PY2, default=_nodefault):
+ def json_load(self, sliceno=None, unicode_as_utf8bytes=False, default=_nodefault):
from accelerator.extras import json_load
try:
return json_load(self.filename(sliceno), unicode_as_utf8bytes=unicode_as_utf8bytes)
diff --git a/accelerator/launch.py b/accelerator/launch.py
index e21fa5f2..3e42ea49 100644
--- a/accelerator/launch.py
+++ b/accelerator/launch.py
@@ -20,9 +20,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-
import os
import signal
import sys
@@ -86,13 +83,13 @@ def call_analysis(analysis_func, sliceno_, delayed_start, q, preserve_result, pa
for fd in output_fds:
os.close(fd)
os.close(_prof_fd)
- slicename = 'analysis(%d)' % (sliceno_,)
+ slicename = f'analysis({sliceno_})'
setproctitle(slicename)
from accelerator.extras import saved_files, _backgrounded_wait
saved_files.clear() # don't inherit (and then return) the files from prepare
if delayed_start:
os.close(delayed_start[1])
- update = statmsg._start('waiting for concurrency limit (%d)' % (sliceno_,), parent_pid, True)
+ update = statmsg._start(f'waiting for concurrency limit ({sliceno_})', parent_pid, True)
if os.read(delayed_start[0], 1) != b'a':
raise AcceleratorError('bad delayed_start, giving up')
update(slicename)
@@ -142,7 +139,7 @@ def save(item, name):
if sliceno_ == 0:
blob.save(len(res), "Analysis.tuple", temp=True)
for ix, item in enumerate(res):
- save(item, "Analysis.%d." % (ix,))
+ save(item, f"Analysis.{ix}.")
else:
if sliceno_ == 0:
blob.save(False, "Analysis.tuple", temp=True)
@@ -187,7 +184,7 @@ def fork_analysis(slices, concurrency, analysis_func, kw, preserve_result, outpu
# The rest will wait on this queue
delayed_start = os.pipe()
delayed_start_todo = slices - i
- p = SimplifiedProcess(target=call_analysis, args=(analysis_func, i, delayed_start, q, preserve_result, pid, output_fds), kwargs=kw, name='analysis-%d' % (i,))
+ p = SimplifiedProcess(target=call_analysis, args=(analysis_func, i, delayed_start, q, preserve_result, pid, output_fds), kwargs=kw, name=f'analysis-{i}')
children.append(p)
for fd in output_fds:
os.close(fd)
@@ -209,7 +206,7 @@ def fork_analysis(slices, concurrency, analysis_func, kw, preserve_result, outpu
else:
exit_count -= 1
if p.exitcode:
- raise AcceleratorError("%s terminated with exitcode %d" % (p.name, p.exitcode,))
+ raise AcceleratorError(f"{p.name} terminated with exitcode {p.exitcode}")
children = still_alive
reap_time = monotonic() + 5
# If a process dies badly we may never get a message here.
@@ -237,7 +234,7 @@ def fork_analysis(slices, concurrency, analysis_func, kw, preserve_result, outpu
if len(finishjob) != 1 and not s_tb:
s_tb = 'not all slices agreed about job.finish_early() in analysis'
if s_tb:
- data = [{'analysis(%d)' % (s_no,): s_tb}, None]
+ data = [{f'analysis({s_no})': s_tb}, None]
writeall(_prof_fd, json.dumps(data).encode('utf-8'))
exitfunction()
if delayed_start_todo:
@@ -289,7 +286,7 @@ def fmt_tb(skip_level):
msg.append(" " * ix)
msg.append(txt)
if line_report and line_report[0]:
- msg.append(' reached line %d' % (line_report[0].count,))
+ msg.append(f' reached line {line_report[0].count}')
msg.append("\n")
msg.extend(format_exception_only(e_type, e))
return ''.join(msg)
@@ -326,7 +323,7 @@ def execute_process(workdir, jobid, slices, concurrency, index=None, workdirs=No
g.server_url = server_url
g.running = 'launch'
- statmsg._start('%s %s' % (jobid, params.method,), parent_pid)
+ statmsg._start(f'{jobid} {params.method}', parent_pid)
def dummy():
pass
diff --git a/accelerator/metadata.py b/accelerator/metadata.py
index 3691d8fd..6da28b1f 100644
--- a/accelerator/metadata.py
+++ b/accelerator/metadata.py
@@ -20,8 +20,6 @@
# Functions for generating, inserting and extracting metadata about the job
# that built files.
-from __future__ import print_function
-
import json
import re
import socket
@@ -30,14 +28,10 @@
import traceback
import zlib
-from accelerator.compat import PY3, FileNotFoundError
+from accelerator.compat import FileNotFoundError
from accelerator.extras import json_decode
-if PY3:
- crc32 = zlib.crc32
-else:
- def crc32(data):
- return zlib.crc32(data) & 0xffffffff
+crc32 = zlib.crc32
def b64hash_setup(filename):
@@ -60,9 +54,7 @@ def job_metadata(job):
'setup_hash': b64hash_setup(job.filename('setup.json')),
'host': socket.gethostname(),
}
- res = json.dumps(d)
- if PY3:
- res = res.encode('ascii')
+ res = json.dumps(d).encode('ascii')
return res
@@ -164,7 +156,7 @@ def chunks(data):
def extract_gif(fh):
def unchunk(pos):
while pos + 1 < len(data):
- z, = struct.unpack('= 512, "POSIX says PIPE_BUF is at least 512, you have %d" % (select.PIPE_BUF,)
+assert select.PIPE_BUF >= 512, f"POSIX says PIPE_BUF is at least 512, you have {select.PIPE_BUF}"
PIPE_BUF = min(select.PIPE_BUF, 65536)
MAX_PART = PIPE_BUF - 6
@@ -248,7 +245,7 @@ def __init__(self, target, args=(), kwargs={}, name=None, stdin=None, ignore_EPI
os.kill(os.getpid(), signal.SIGINT)
except Exception as e:
if not isinstance(e, OSError) or e.errno != errno.EPIPE:
- print("Exception in %d %r:" % (os.getpid(), name), file=sys.stderr)
+ print(f"Exception in {os.getpid()} {name!r}:", file=sys.stderr)
print_exc(file=sys.stderr)
elif ignore_EPIPE:
rc = 0
diff --git a/accelerator/runner.py b/accelerator/runner.py
index 5f9435da..2fd93b1f 100644
--- a/accelerator/runner.py
+++ b/accelerator/runner.py
@@ -25,9 +25,6 @@
# Also contains the function that starts these (new_runners) and the dict
# of running ones {version: Runner}
-from __future__ import print_function
-from __future__ import division
-
from importlib import import_module
from types import ModuleType
from traceback import print_exc
@@ -80,7 +77,7 @@ def check_picklable(desc, value):
return
except Exception as e:
msg = str(e)
- raise MsgException('Unpicklable %s: %s' % (desc, msg,))
+ raise MsgException(f'Unpicklable {desc}: {msg}')
def load_methods(all_packages, data):
from accelerator.compat import str_types, iteritems
@@ -110,7 +107,7 @@ def tar_add(name, data):
except Exception:
pass
for package, key in data:
- modname = '%s.a_%s' % (package, key)
+ modname = f'{package}.a_{key}'
try:
mod, mod_filename, prefix = get_mod(modname)
depend_extra = []
@@ -122,7 +119,7 @@ def tar_add(name, data):
dep = prefix + dep
depend_extra.append(dep)
else:
- raise MsgException('Bad depend_extra: %r' % (dep,))
+ raise MsgException(f'Bad depend_extra: {dep!r}')
dep_prefix = os.path.commonprefix(depend_extra + [mod_filename])
# commonprefix works per character (and commonpath is v3.5+)
dep_prefix = dep_prefix.rsplit('/', 1)[0] + '/'
@@ -152,8 +149,8 @@ def tar_add(name, data):
hash_extra ^= int(hashlib.sha1(data).hexdigest(), 16)
tar_add(dep, data)
for dep in (likely_deps - set(depend_extra)):
- res_warnings.append('%s.a_%s should probably depend_extra on %s' % (package, key, dep_names[dep],))
- res_hashes[key] = ("%040x" % (hash ^ hash_extra,),)
+ res_warnings.append(f'{package}.a_{key} should probably depend_extra on {dep_names[dep]}')
+ res_hashes[key] = (f"{hash ^ hash_extra:040x}",)
res_params[key] = params = DotDict()
# It would have been nice to be able to use ast.get_source_segment
def find_source(name):
@@ -190,7 +187,7 @@ def find_end(startchar, endchar):
end = find_end(b'['[0], b']'[0])
break
if not end:
- print('Failed to figure out where %s is in %s' % (name, key,))
+ print(f'Failed to figure out where {name} is in {key}')
end = start
return slice(start, end)
except Exception:
@@ -207,23 +204,23 @@ def fmtopt(v):
elif isinstance(v, dict):
return '{%s}' % (', '.join('%s: %s' % (fmtopt(k), fmtopt(v)) for k, v in v.items()),)
elif isinstance(v, list):
- return '[%s]' % (', '.join(fmtopt(v) for v in v),)
+ return f"[{', '.join((fmtopt(v) for v in v))}]"
elif isinstance(v, OptionEnum):
return '{%s}' % (', '.join(sorted(map(str, v._valid))),)
elif isinstance(v, OptionEnumValue):
return '%r {%s}' % (v, ', '.join(sorted(map(str, v._valid))),)
elif isinstance(v, RequiredOption):
- return 'RequiredOption(%s%s)' % (fmtopt(v.value), ', none_ok=True' if v.none_ok else '',)
+ return f"RequiredOption({fmtopt(v.value)}{', none_ok=True' if v.none_ok else ''})"
elif isinstance(v, OptionDefault):
if v.default is None:
- return 'OptionDefault(%s)' % (fmtopt(v.value),)
- return 'OptionDefault(%s, default=%s)' % (fmtopt(v.value), fmtopt(v.default),)
+ return f'OptionDefault({fmtopt(v.value)})'
+ return f'OptionDefault({fmtopt(v.value)}, default={fmtopt(v.default)})'
else:
return repr(v)
for name, default in (('options', {},), ('datasets', (),), ('jobs', (),),):
params[name] = d = getattr(mod, name, default)
if not isinstance(d, type(default)):
- raise MsgException("%s should be a %s" % (name, type(default).__name__,))
+ raise MsgException(f"{name} should be a {type(default).__name__}")
if d:
items = {v[0] if isinstance(v, list) else v for v in params[name]}
if isinstance(d, dict):
@@ -271,7 +268,7 @@ def fmtopt(v):
d = res_descriptions[key].get(name)
for item in getattr(mod, name, ()):
if isinstance(item, list):
- d['[%s]' % (item[0],)] = d.pop(item[0])
+ d[f'[{item[0]}]'] = d.pop(item[0])
equivalent_hashes = getattr(mod, 'equivalent_hashes', ())
if equivalent_hashes:
try:
@@ -293,11 +290,11 @@ def fmtopt(v):
end -= 1 # to get the same hash as the old way of parsing
h = hashlib.sha1(src[:start])
h.update(src[end:])
- verifier = "%040x" % (int(h.hexdigest(), 16) ^ hash_extra,)
+ verifier = f"{int(h.hexdigest(), 16) ^ hash_extra:040x}"
if verifier == k:
res_hashes[key] += v
else:
- res_warnings.append('%s.a_%s has equivalent_hashes, but missing verifier %s' % (package, key, verifier,))
+ res_warnings.append(f'{package}.a_{key} has equivalent_hashes, but missing verifier {verifier}')
tar_o.close()
tar_fh.seek(0)
archives[key] = tar_fh.read()
@@ -305,7 +302,7 @@ def fmtopt(v):
check_picklable('description', res_descriptions[key])
except Exception as e:
if isinstance(e, MsgException):
- print('%s: %s' % (modname, str(e),))
+ print(f'{modname}: {e!s}')
else:
print_exc(file=sys.stderr)
res_failed.append(modname)
@@ -316,10 +313,7 @@ def fmtopt(v):
def launch_start(data):
from accelerator.launch import run
- from accelerator.compat import PY2
from accelerator.dispatch import close_fds
- if PY2:
- data = {k: v.encode('utf-8') if isinstance(v, unicode) else v for k, v in data.items()}
prof_r, prof_w = os.pipe()
# Disable the GC here, leaving it disabled in the child (the method).
# The idea is that most methods do not actually benefit from the GC, but
@@ -408,7 +402,7 @@ def __init__(self, pid, sock, python):
self._lock = TLock()
self._thread = Thread(
target=self._receiver,
- name="%d receiver" % (pid,),
+ name=f"{pid} receiver",
)
self._thread.daemon = True
self._thread.start()
@@ -560,9 +554,9 @@ def new_runners(config, used_versions):
pass
r1, r2 = resource.getrlimit(resource.RLIMIT_NOFILE)
if r1 < r2:
- print("WARNING: Failed to raise RLIMIT_NOFILE to %d. Set to %d." % (r2, r1,))
+ print(f"WARNING: Failed to raise RLIMIT_NOFILE to {r2}. Set to {r1}.")
if r1 < 5000:
- print("WARNING: RLIMIT_NOFILE is %d, that's not much." % (r1,))
+ print(f"WARNING: RLIMIT_NOFILE is {r1}, that's not much.")
wait_for = []
try:
diff --git a/accelerator/server.py b/accelerator/server.py
index c487c97d..6848807b 100644
--- a/accelerator/server.py
+++ b/accelerator/server.py
@@ -19,9 +19,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-
import sys
import socket
import traceback
@@ -36,7 +33,7 @@
import random
import atexit
-from accelerator.compat import unicode, monotonic
+from accelerator.compat import monotonic
from accelerator.web import ThreadedHTTPServer, ThreadedUnixHTTPServer, BaseWebHandler
@@ -80,12 +77,12 @@ def do_response(self, code, content_type, body):
def encode_body(self, body):
if isinstance(body, bytes):
return body
- if isinstance(body, unicode):
+ if isinstance(body, str):
return body.encode('utf-8')
return json_encode(body)
def handle_req(self, path, args):
- if self.DEBUG: print("@server.py: Handle_req, path = \"%s\", args = %s" %( path, args ), file=sys.stderr)
+ if self.DEBUG: print(f"@server.py: Handle_req, path = \"{path}\", args = {args}", file=sys.stderr)
try:
self._handle_req( path, args )
except Exception:
@@ -188,11 +185,11 @@ def _handle_req(self, path, args):
else:
start_ix = len(jobs) - 1
if start_ix is None:
- res = {'error': '%s is not a %s %s job' % (start_from, typ, method,)}
+ res = {'error': f'{start_from} is not a {typ} {method} job'}
else:
num = int(args.get('offset', 0))
if not jobs:
- res = {'error': 'no %s jobs with method %s available' % (typ, method,)}
+ res = {'error': f'no {typ} jobs with method {method} available'}
elif 0 <= start_ix + num < len(jobs):
res = {'id': jobs[start_ix + num]}
else:
@@ -202,7 +199,7 @@ def _handle_req(self, path, args):
else:
direction, kind = 'forward', 'later'
available = len(jobs) - start_ix - 1
- res = {'error': 'tried to go %d jobs %s from %s, but only %d %s %s %s jobs available' % (abs(num), direction, jobs[start_ix], available, kind, typ, method,)}
+ res = {'error': f'tried to go {abs(num)} jobs {direction} from {jobs[start_ix]}, but only {available} {kind} {typ} {method} jobs available'}
self.do_response(200, 'text/json', res)
elif path[0] == 'job_is_current':
@@ -223,7 +220,7 @@ def _handle_req(self, path, args):
job = self.ctrl.workspaces[workdir].allocate_jobs(1)[0]
self.do_response(200, 'text/json', {'jobid': job})
else:
- self.do_response(500, 'text/json', {'error': 'workdir %r does not exist' % (workdir,)})
+ self.do_response(500, 'text/json', {'error': f'workdir {workdir!r} does not exist'})
elif path==['submit']:
if self.ctrl.broken:
@@ -357,7 +354,7 @@ def exitfunction(*a):
signal.signal(signal.SIGTERM, signal.SIG_IGN)
signal.signal(signal.SIGINT, signal.SIG_IGN)
print()
- print('The deathening! %d %s' % (os.getpid(), children,))
+ print(f'The deathening! {os.getpid()} {children}')
print()
for child in list(children):
try:
@@ -386,7 +383,7 @@ def run(self):
except Exception:
traceback.print_exc(file=sys.stderr)
finally:
- print("Thread %r died. That's bad." % (self.name,))
+ print(f"Thread {self.name!r} died. That's bad.")
exitfunction(DeadlyThread)
@@ -409,7 +406,7 @@ def check_socket(fn):
except OSError:
pass
return
- raise Exception("Socket %s already listening" % (fn,))
+ raise Exception(f"Socket {fn} already listening")
def siginfo(sig, frame):
print_status_stacks()
@@ -512,15 +509,15 @@ def buf_up(fh, opt):
for n in ("project_directory", "result_directory", "input_directory",):
v = config.get(n)
n = n.replace("_", " ")
- print("%17s: %s" % (n, v,))
+ print(f"{n:>17}: {v}")
for n in ("board", "urd",):
v = config.get(n + '_listen')
if v and not config.get(n + '_local', True):
extra = ' (remote)'
else:
extra = ''
- print("%17s: %s%s" % (n, v, extra,))
+ print(f"{n:>17}: {v}{extra}")
print()
- print("Serving on %s\n" % (config.listen,), file=sys.stderr)
+ print(f"Serving on {config.listen}\n", file=sys.stderr)
server.serve_forever()
diff --git a/accelerator/setupfile.py b/accelerator/setupfile.py
index 35306ca7..26d5d491 100644
--- a/accelerator/setupfile.py
+++ b/accelerator/setupfile.py
@@ -20,15 +20,12 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-
from collections import OrderedDict
from json import dumps
from datetime import datetime, date, time, timedelta
from pathlib import PosixPath, PurePosixPath
-from accelerator.compat import iteritems, unicode, long, PY3, PY2, uni
+from accelerator.compat import iteritems, uni
from accelerator.error import AcceleratorError, NoSuchJobError
from accelerator.extras import DotDict, json_load, json_save, json_encode
@@ -65,7 +62,7 @@ def load_setup(jobid):
try:
d = json_load('setup.json', jobid)
except IOError:
- raise NoSuchJobError('Job %r not found' % (jobid,))
+ raise NoSuchJobError(f'Job {jobid!r} not found')
version = d.version
if version == 1:
d.jobs = d.pop('jobids')
@@ -82,7 +79,7 @@ def load_setup(jobid):
# no changes here, it's only used to know how to find datasets
version = 4
if version != 4:
- raise AcceleratorError("Don't know how to load setup.json version %d (in %s)" % (d.version, jobid,))
+ raise AcceleratorError(f"Don't know how to load setup.json version {d.version} (in {jobid})")
return d
def update_setup(jobid, **kw):
@@ -129,10 +126,8 @@ def copy_json_safe(src):
return [1970, 1, 1, src.hour, src.minute, src.second, src.microsecond]
elif isinstance(src, timedelta):
return src.total_seconds()
- elif PY2 and isinstance(src, bytes):
- return uni(src)
else:
- assert isinstance(src, (str, unicode, int, float, long, bool)) or src is None, "%s not supported in data" % (type(src),)
+ assert isinstance(src, (str, int, float, bool)) or src is None, f"{type(src)} not supported in data"
return src
def encode_setup(data, sort_keys=True, as_str=False):
@@ -141,7 +136,7 @@ def encode_setup(data, sort_keys=True, as_str=False):
compact_keys=('starttime', 'endtime', 'exectime', '_typing',),
special_keys=('options', 'datasets', 'jobs',),
)
- if PY3 and not as_str:
+ if not as_str:
res = res.encode('ascii')
return res
@@ -167,7 +162,7 @@ def _encode_with_compact(data, compact_keys, extra_indent=0, separator='\n', spe
fmted = _encode_with_compact(d, ('analysis', 'per_slice',), 1, '')
else:
fmted = dumps(data[k])
- compact.append(' "%s": %s,' % (k, fmted,))
+ compact.append(f' "{k}": {fmted},')
del data[k]
for k in special_keys:
if k in data:
diff --git a/accelerator/shell/__init__.py b/accelerator/shell/__init__.py
index cd2388b4..886cd4db 100644
--- a/accelerator/shell/__init__.py
+++ b/accelerator/shell/__init__.py
@@ -18,10 +18,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
import sys
import errno
import os
@@ -83,13 +79,13 @@ def load_some_cfg(basedir='.', all=False):
# As long as we find at least one we're happy.
pass
if not found_any:
- raise UserError("Could not find 'accelerator*.conf' in %r or any of its parents." % (basedir,))
+ raise UserError(f"Could not find 'accelerator*.conf' in {basedir!r} or any of its parents.")
cfg.config_filename = None
else:
try:
fn = next(cfgs)
except StopIteration:
- raise UserError("Could not find 'accelerator.conf' in %r or any of its parents." % (basedir,))
+ raise UserError(f"Could not find 'accelerator.conf' in {basedir!r} or any of its parents.")
load_cfg(fn)
def load_cfg(fn):
@@ -101,7 +97,7 @@ def load_cfg(fn):
cfg = load_config(fn)
for k, v in cfg.workdirs.items():
if WORKDIRS.get(k, v) != v:
- print("WARNING: %s overrides workdir %s" % (fn, k,), file=sys.stderr)
+ print(f"WARNING: {fn} overrides workdir {k}", file=sys.stderr)
WORKDIRS[k] = v
return cfg
@@ -193,7 +189,7 @@ def cmd_abort(argv):
a = Automata(cfg.url)
res = a.abort()
if not args.quiet:
- print("Killed %d running job%s." % (res.killed, '' if res.killed == 1 else 's'))
+ print(f"Killed {res.killed} running job{'' if res.killed == 1 else 's'}.")
cmd_abort.help = '''abort running job(s)'''
def cmd_alias(argv):
@@ -273,7 +269,7 @@ def cmd_version(argv, as_command=True):
py_version = sys.implementation.name
suffix = ' (%s)' % (suffix.split('\n')[0].strip(),)
impl_version = '.'.join(map(str, sys.implementation.version))
- py_version = '%s %s' % (py_version, impl_version,)
+ py_version = f'{py_version} {impl_version}'
except Exception:
pass
print('Running on ' + py_version + suffix)
@@ -308,7 +304,6 @@ def _unesc(m):
return _unesc_v.get(v.lower(), v)
def parse_user_config():
- from accelerator.compat import open
from configparser import ConfigParser
from os import environ
fns = []
@@ -342,7 +337,7 @@ def read(fn, seen_fns=(), must_exist=False):
for include in shlex.split(include):
include = join(dirname(fn), expanduser(include))
if include in seen_fns:
- raise Exception('Config include loop: %r goes back to %r' % (fn, include,))
+ raise Exception(f'Config include loop: {fn!r} goes back to {include!r}')
read(include, seen_fns, True)
# Append this file after the included files, so this file takes priority.
all_cfg.append((fn, contents))
@@ -365,7 +360,7 @@ def chopline(description, max_len):
if len(description) + len(part) + 1 > max_len:
break
if description:
- description = '%s %s' % (description, part,)
+ description = f'{description} {part}'
else:
description = part
description += colour.faint(ddot)
@@ -413,7 +408,7 @@ def expand_env(words, alias):
try:
expanded = shlex.split(environ[k])
except ValueError as e:
- raise ValueError('Failed to expand alias %s (%s -> %r): %s' % (alias, word, environ[k], e,))
+ raise ValueError(f'Failed to expand alias {alias} ({word} -> {environ[k]!r}): {e}')
for word in expanded:
yield word
else:
@@ -428,7 +423,7 @@ def expand_aliases(main_argv, argv):
try:
expanded = shlex.split(aliases[alias])
except ValueError as e:
- raise ValueError('Failed to expand alias %s (%r): %s' % (argv[0], aliases[argv[0]], e,))
+ raise ValueError(f'Failed to expand alias {argv[0]} ({aliases[argv[0]]!r}): {e}')
expanded = list(expand_env(expanded, alias))
more_main_argv, argv = split_args(expanded + argv[1:])
main_argv.extend(more_main_argv)
@@ -436,7 +431,7 @@ def expand_aliases(main_argv, argv):
break
used_aliases.append(alias)
if alias in used_aliases[:-1]:
- raise ValueError('Alias loop: %r' % (used_aliases,))
+ raise ValueError(f'Alias loop: {used_aliases!r}')
while argv and argv[0] == 'noalias':
argv.pop(0)
@@ -539,7 +534,7 @@ def main():
parser.print_help(file=sys.stderr)
if args.command is not None:
print(file=sys.stderr)
- print('Unknown command "%s"' % (args.command,), file=sys.stderr)
+ print(f'Unknown command "{args.command}"', file=sys.stderr)
sys.exit(2)
config_fn = args.config
if args.command in ('init', 'intro', 'version', 'alias',):
@@ -548,7 +543,7 @@ def main():
debug_cmd = getattr(cmd, 'is_debug', False)
try:
setup(config_fn, debug_cmd)
- argv.insert(0, '%s %s' % (basename(sys.argv[0]), args.command,))
+ argv.insert(0, f'{basename(sys.argv[0])} {args.command}')
return cmd(argv)
except UserError as e:
print(e, file=sys.stderr)
diff --git a/accelerator/shell/ds.py b/accelerator/shell/ds.py
index b633472b..2164a047 100644
--- a/accelerator/shell/ds.py
+++ b/accelerator/shell/ds.py
@@ -19,8 +19,6 @@
# #
############################################################################
-from __future__ import division, print_function
-
import sys
import locale
from math import ceil
@@ -74,7 +72,7 @@ def format_location(loc):
is_local, ds, colname = loc
if is_local:
return 'local'
- return '%s in %s' % (quote(colname), ds.quoted,)
+ return f'{quote(colname)} in {ds.quoted}'
def typed_from(ds, loc):
if not loc:
@@ -87,7 +85,7 @@ def typed_from(ds, loc):
res = 'typed from ' + format_location((False, src_ds, colname))
orig_loc = original_location(src_ds, colname)
if orig_loc and not orig_loc[0]:
- return '%s, originally %s' % (res, format_location(orig_loc))
+ return f'{res}, originally {format_location(orig_loc)}'
else:
return res
@@ -118,7 +116,7 @@ def finish(badinput):
if badinput and not args.suppress_errors:
print('Error, failed to resolve datasets:', file=sys.stderr)
for n, e in badinput:
- print(' %r: %s' % (n, e,), file=sys.stderr)
+ print(f' {n!r}: {e}', file=sys.stderr)
exit(1)
exit()
@@ -135,7 +133,7 @@ def finish(badinput):
badinput.append((n, e))
dsvec = None
if dsvec:
- print('%s' % (dsvec[0].job,))
+ print(f'{dsvec[0].job}')
v = []
for ds in dsvec:
if args.chainedlist:
@@ -196,7 +194,7 @@ def fmt_minmax(val):
return str(val)
minlen = max(len(fmt_minmax(chain.min(n))) for n in ds.columns)
maxlen = max(len(fmt_minmax(chain.max(n))) for n in ds.columns)
- minmax_template = '[%%%ds, %%%ds]' % (min(18, minlen), min(18, maxlen),)
+ minmax_template = f'[%{min(18, minlen)}s, %{min(18, maxlen)}s]'
def prettyminmax(n):
minval, maxval = chain.min(n), chain.max(n)
if args.suppress_minmax or minval is None:
@@ -214,8 +212,8 @@ def prettyminmax(n):
except Exception:
# source job might be deleted
pass
- print(" {0} columns".format(fmt_num(len(ds.columns))))
- print(" {0} lines".format(fmt_num(sum(ds.lines))))
+ print(f" {fmt_num(len(ds.columns))} columns")
+ print(f" {fmt_num(sum(ds.lines))} lines")
if ds.previous or args.chain:
chain = ds.chain()
@@ -225,15 +223,15 @@ def prettyminmax(n):
if in_job == len(chain):
in_job = ' (all within job)'
else:
- in_job = ' ({0} within job)'.format(fmt_num(in_job))
+ in_job = f' ({fmt_num(in_job)} within job)'
else:
in_job = ''
- print(" {0} length {1}{2}, from {3} to {4}".format(full_name, fmt_num(len(chain)), in_job, chain[0], chain[-1]))
+ print(f" {full_name} length {fmt_num(len(chain))}{in_job}, from {chain[0]} to {chain[-1]}")
if args.non_empty_chain:
chain = [ds for ds in chain if sum(ds.lines)]
- print(" Filtered chain length {0}".format(fmt_num(len(chain))))
+ print(f" Filtered chain length {fmt_num(len(chain))}")
if args.chain:
- data = tuple((ix, "%s/%s" % (x.job, x.name), fmt_num(sum(x.lines))) for ix, x in enumerate(chain))
+ data = tuple((ix, f"{x.job}/{x.name}", fmt_num(sum(x.lines))) for ix, x in enumerate(chain))
max_n, max_l = colwidth(x[1:] for x in data)
template = "{0:3}: {1:%d} ({2:>%d})" % (max_n, max_l)
printcolwise(data, template, lambda x: (x[0], x[1], x[2]), minrows=8, indent=8)
@@ -256,6 +254,6 @@ def prettyminmax(n):
print(" Max to average ratio: " + locale.format_string("%2.3f", (max(x[2] for x in data) / ((s or 1e20) / len(data)),) ))
if ds.previous:
- print(" {0} total lines in chain".format(fmt_num(sum(sum(ds.lines) for ds in chain))))
+ print(f" {fmt_num(sum((sum(ds.lines) for ds in chain)))} total lines in chain")
finish(badinput)
diff --git a/accelerator/shell/gc.py b/accelerator/shell/gc.py
index 292214b3..06ed9133 100644
--- a/accelerator/shell/gc.py
+++ b/accelerator/shell/gc.py
@@ -17,8 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-
from accelerator.error import AcceleratorError
from accelerator.job import Job
from accelerator.shell.parser import ArgumentParser
@@ -50,8 +48,8 @@ def candidate(jid):
print("Nothing to do.")
return 0
if args.dry_run:
- print("Would have deleted %d jobs" % (len(to_delete),))
+ print(f"Would have deleted {len(to_delete)} jobs")
return 0
- print("Deleting %d jobs" % (len(to_delete),))
+ print(f"Deleting {len(to_delete)} jobs")
for job in to_delete:
shutil.rmtree(job.path)
diff --git a/accelerator/shell/grep.py b/accelerator/shell/grep.py
index a293f8e7..4faf2446 100644
--- a/accelerator/shell/grep.py
+++ b/accelerator/shell/grep.py
@@ -21,8 +21,6 @@
# grep in a dataset(chain)
-from __future__ import division, print_function
-
import sys
import re
import os
@@ -36,7 +34,7 @@
import operator
import signal
-from accelerator.compat import unicode, izip, PY2
+from accelerator.compat import izip
from accelerator.compat import izip_longest
from accelerator.compat import monotonic
from accelerator.compat import num_types
@@ -69,7 +67,7 @@ def number_or_None(obj):
# Base 16 has to be handled separately, because using 0 will
# error on numbers starting with 0 (on python 3).
# But we have to check for 0x, so things like "a" are not accepted.
- if (isinstance(obj, unicode) and '0x' in obj) or (isinstance(obj, bytes) and b'0x' in obj):
+ if (isinstance(obj, str) and '0x' in obj) or (isinstance(obj, bytes) and b'0x' in obj):
try:
return int(obj, 16)
except ValueError:
@@ -78,7 +76,7 @@ def number_or_None(obj):
def number_or_error(obj):
number = number_or_None(obj)
if number is None:
- raise re.error('%r is not a valid not a number' % (obj,))
+ raise re.error(f'{obj!r} is not a valid not a number')
return number
def splitprefix(s, prefixes):
@@ -179,14 +177,14 @@ def __call__(self, parser, namespace, values, option_string=None):
else:
name = next(unnamed)
if name not in names:
- raise ArgumentError(self, 'unknown field %r' % (name,))
+ raise ArgumentError(self, f'unknown field {name!r}')
name = names[name]
try:
value = int(value)
except ValueError:
- raise ArgumentError(self, 'invalid int value for %s: %r' % (name, value,))
+ raise ArgumentError(self, f'invalid int value for {name}: {value!r}')
if value < min_value[name] or value > 9999:
- raise ArgumentError(self, 'invalid value for %s: %d' % (name, value,))
+ raise ArgumentError(self, f'invalid value for {name}: {value}')
tab_length[name] = value
# -T overrides -t
namespace.separator = None
@@ -241,7 +239,7 @@ def __call__(self, parser, namespace, values, option_string=None):
"TABLEN works like normal tabs\n" +
"FIELDLEN sets a longer minimum between fields\n" +
"MINLEN sets a minimum len for all separators\n" +
- "use \"-T/\" to just activate it (sets %d/%d/%d)" % (tab_length.tab_len, tab_length.field_len, tab_length.min_len,)
+ f"use \"-T/\" to just activate it (sets {tab_length.tab_len}/{tab_length.field_len}/{tab_length.min_len})"
)
parser.add_argument('-B', '--before-context', type=int, default=0, metavar='NUM', help="print NUM lines of leading context", )
parser.add_argument('-A', '--after-context', type=int, default=0, metavar='NUM', help="print NUM lines of trailing context", )
@@ -316,7 +314,7 @@ def __call__(self, parser, namespace, values, option_string=None):
else:
patterns.append(re.compile(pattern, re_flags))
except re.error as e:
- print("Bad pattern %r:\n%s" % (pattern, e,), file=sys.stderr)
+ print(f"Bad pattern {pattern!r}:\n{e}", file=sys.stderr)
return 1
grep_columns = set(args.grep or ())
@@ -327,7 +325,7 @@ def __call__(self, parser, namespace, values, option_string=None):
if args.slice:
want_slices = []
for s in args.slice:
- assert 0 <= s < g.slices, "Slice %d not available" % (s,)
+ assert 0 <= s < g.slices, f"Slice {s} not available"
if s not in want_slices:
want_slices.append(s)
else:
@@ -387,7 +385,7 @@ def columns_for_ds(ds, columns=columns, not_columns=not_columns):
for ds in datasets:
missing = need_cols - set(ds.columns)
if missing:
- print('ERROR: %s does not have columns %r' % (ds, missing,), file=sys.stderr)
+ print(f'ERROR: {ds} does not have columns {missing!r}', file=sys.stderr)
bad = True
if bad:
return 1
@@ -512,7 +510,7 @@ def escape_item(item):
return item.replace('\\', '\\\\').replace('\n', '\\n')
else:
escape_item = None
- errors = 'replace' if PY2 else 'surrogateescape'
+ errors = 'surrogateescape'
if args.unique:
# A --unique without a value means all, and deletes any previously specified columns.
@@ -589,27 +587,27 @@ def show(sig, frame):
if ds_ix == len(datasets):
msg = 'DONE'
else:
- msg = '{0:d}% of {1:n} lines'.format(round(p * 100), total_lines_per_slice_at_ds[-1][sliceno])
+ msg = f'{round(p * 100):d}% of {total_lines_per_slice_at_ds[-1][sliceno]:n} lines'
if show_ds:
- msg = '%s (in %s)' % (msg, datasets[ds_ix].quoted,)
- msg = '%9d: %s' % (sliceno, msg,)
+ msg = f'{msg} (in {datasets[ds_ix].quoted})'
+ msg = f'{sliceno:9}: {msg}'
if p < bad_cutoff:
msg = colour(msg, 'grep/infohighlight')
else:
msg = colour(msg, 'grep/info')
write(2, msg.encode('utf-8') + b'\n')
- msg = '{0:d}% of {1:n} lines'.format(round(progress_total * 100), total_lines)
+ msg = f'{round(progress_total * 100):d}% of {total_lines:n} lines'
if len(datasets) > 1:
min_ds = min(ds_ixes)
max_ds = max(ds_ixes)
if min_ds < len(datasets):
ds_name = datasets[min_ds].quoted
extra = '' if min_ds == max_ds else ' ++'
- msg = '%s (in %s%s)' % (msg, ds_name, extra,)
+ msg = f'{msg} (in {ds_name}{extra})'
worst = min(progress_fraction)
if worst < bad_cutoff:
- msg = '%s, worst %d%%' % (msg, round(worst * 100),)
- msg = colour(' SUMMARY: %s' % (msg,), 'grep/info')
+ msg = f'{msg}, worst {round(worst * 100)}%'
+ msg = colour(f' SUMMARY: {msg}', 'grep/info')
write(2, msg.encode('utf-8') + b'\n')
for signame in ('SIGINFO', 'SIGUSR1'):
if hasattr(signal, signame):
@@ -1039,9 +1037,9 @@ def show(lineno, items):
if args.show_sliceno and args.roundrobin:
(prefix['sliceno'], lineno), items = items
if only_matching == 'part':
- items = [filter_item(unicode(item)) for item in items]
+ items = [filter_item(str(item)) for item in items]
if only_matching == 'columns':
- d = {k: v for k, v in zip(used_columns, items) if filter_item(unicode(v))}
+ d = {k: v for k, v in zip(used_columns, items) if filter_item(str(v))}
else:
d = dict(zip(used_columns, items))
if args.show_lineno:
@@ -1063,10 +1061,10 @@ def show(lineno, items):
data = list(prefix)
if args.show_sliceno and args.roundrobin:
(sliceno, lineno), items = items
- data[-1] = unicode(sliceno)
+ data[-1] = str(sliceno)
if args.show_lineno:
- data.append(unicode(lineno))
- show_items = map(unicode, items)
+ data.append(str(lineno))
+ show_items = map(str, items)
if only_matching:
if only_matching == 'columns':
show_items = (item if filter_item(item) else '' for item in show_items)
@@ -1129,7 +1127,7 @@ def cb(n):
else:
it = ds._column_iterator(sliceno, col, **kw)
if ds.columns[col].type == 'bytes':
- errors = 'replace' if PY2 else 'surrogateescape'
+ errors = 'surrogateescape'
if ds.columns[col].none_support:
it = (None if v is None else v.decode('utf-8', errors) for v in it)
else:
@@ -1183,7 +1181,7 @@ def mk_slicelineno_iter():
if args.numeric:
fmtfix = number_or_None
else:
- fmtfix = unicode
+ fmtfix = str
if args.unique:
if args.unique is True: # all columns
care_mask = [True] * len(used_columns)
@@ -1331,7 +1329,7 @@ def gen_headers(headers):
unique_columns = tuple(col for col in columns_for_ds(ds) if unique_filter(col))
if check_unique_columns_existance and len(unique_columns) != len(args.unique):
missing = args.unique - set(unique_columns)
- print('ERROR: %s does not have columns %r' % (ds.quoted, missing,), file=sys.stderr)
+ print(f'ERROR: {ds.quoted} does not have columns {missing!r}', file=sys.stderr)
bad = True
if unique_columns not in unique_columns2ix:
unique_columns2ix[unique_columns] = unique_columns_ix
@@ -1364,7 +1362,7 @@ def gen_headers(headers):
p = mp.SimplifiedProcess(
target=one_slice,
args=(sliceno, q_in, q_out, q_to_close,),
- name='slice-%d' % (sliceno,),
+ name=f'slice-{sliceno}',
ignore_EPIPE=bool(liner),
)
children.append(p)
diff --git a/accelerator/shell/hist.py b/accelerator/shell/hist.py
index e94752bf..4a514109 100644
--- a/accelerator/shell/hist.py
+++ b/accelerator/shell/hist.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
from accelerator.colourwrapper import colour
from accelerator.compat import fmt_num, num_types, int_types
from accelerator.error import NoSuchWhateverError
@@ -168,7 +164,7 @@ def main(argv, cfg):
for ds in chain:
for col in args.column:
if col not in ds.columns:
- print("Dataset %s does not have column %s." % (ds.quoted, col,), file=sys.stderr)
+ print(f"Dataset {ds.quoted} does not have column {col}.", file=sys.stderr)
ok = False
if not ok:
return 1
@@ -268,12 +264,12 @@ def name(ix):
if a == b:
return fmt_num(a)
else:
- return '%s - %s' % (fmt_num(a), fmt_num(b))
+ return f'{fmt_num(a)} - {fmt_num(b)}'
else:
def name(ix):
a = step * ix + low
b = step * (ix + 1) + low
- return '%s - %s' % (fmt_num(a), fmt_num(b))
+ return f'{fmt_num(a)} - {fmt_num(b)}'
bin_names = [name(ix) for ix in range(args.max_count)]
def collect_hist(hist, part):
@@ -299,7 +295,7 @@ def collect_hist(hist, part):
total_found = 0 # so we don't print about it later
if args.max_count:
if NaN in hist:
- print(colour('WARNING: Ignored %d NaN values.' % (hist[NaN],), 'hist/warning'), file=sys.stderr)
+ print(colour(f'WARNING: Ignored {hist[NaN]} NaN values.', 'hist/warning'), file=sys.stderr)
hist[args.max_count - 1] += hist[args.max_count] # top value should not be in a separate bin
hist, fmt = formatter([(name, hist[ix]) for ix, name in enumerate(bin_names)])
else:
diff --git a/accelerator/shell/init.py b/accelerator/shell/init.py
index 04a2f065..7d86b4ad 100644
--- a/accelerator/shell/init.py
+++ b/accelerator/shell/init.py
@@ -19,10 +19,6 @@
############################################################################
-from __future__ import print_function
-from __future__ import division
-
-
a_example = r"""description = r'''
This is just an example. It doesn't even try to do anything useful.
@@ -127,7 +123,7 @@ def free(port):
for port in ports:
if all(free(port + n) for n in range(count)):
return port
- raise Exception('Failed to find %d consecutive free TCP ports on %s in range(%d, %d)' % (count, hostname, low, high))
+ raise Exception(f'Failed to find {count} consecutive free TCP ports on {hostname} in range({low}, {high})')
def git(method_dir):
@@ -201,7 +197,7 @@ def add_argument(name, default=None, help='', **kw):
if default.lower() == 'true':
bool_fallbacks.add(cfg_name.replace('-', '_'))
elif default.lower() != 'false':
- raise UserError("Configuration init.%s must be either true or false, not %r." % (cfg_name, default,))
+ raise UserError(f"Configuration init.{cfg_name} must be either true or false, not {default!r}.")
default = None
elif 'type' in kw:
default = kw['type'](default)
@@ -248,9 +244,9 @@ def add_argument(name, default=None, help='', **kw):
else:
port = find_free_ports(0x3000, 0x8000)
listen = DotDict(
- server='%s:%d' % (host, port,),
- board='%s:%d' % (host, port + 1,),
- urd='%s:%d' % (host, port + 2,),
+ server=f'{host}:{port}',
+ board=f'{host}:{port + 1}',
+ urd=f'{host}:{port + 2}',
)
if options.slices is None:
@@ -315,7 +311,7 @@ def slice_count(workdir):
if not options.force:
if exists(options.directory) and set(listdir(options.directory)) - {'venv', '.venv'}:
- raise UserError('Directory %r is not empty.' % (options.directory,))
+ raise UserError(f'Directory {options.directory!r} is not empty.')
def plausible_jobdir(n):
parts = n.rsplit('-', 1)
return len(parts) == 2 and parts[0] == options.name and parts[1].isnumeric()
@@ -324,9 +320,9 @@ def plausible_jobdir(n):
if exists(workdir):
workdir_slices = slice_count(workdir)
if workdir_slices not in (None, options.slices):
- raise UserError('Workdir %r has %d slices, refusing to continue with %d slices' % (workdir, workdir_slices, options.slices,))
+ raise UserError(f'Workdir {workdir!r} has {workdir_slices} slices, refusing to continue with {options.slices} slices')
if exists(first_workdir_path) and any(map(plausible_jobdir, listdir(first_workdir_path))):
- raise UserError('Workdir %r already has jobs in it.' % (first_workdir_path,))
+ raise UserError(f'Workdir {first_workdir_path!r} already has jobs in it.')
if not exists(options.directory):
makedirs(options.directory)
@@ -342,7 +338,7 @@ def plausible_jobdir(n):
makedirs(path)
if options.force or not exists(slices_conf(path)):
with open(slices_conf(path), 'w') as fh:
- fh.write('%d\n' % (options.slices,))
+ fh.write(f'{options.slices}\n')
method_dir = options.name
if not exists(method_dir):
makedirs(method_dir)
@@ -361,7 +357,7 @@ def plausible_jobdir(n):
examples = 'examples'
else:
examples = '# accelerator.examples'
- all_workdirs = ['%s %s' % (shell_quote(name), shell_quote(path),) for name, path in workdirs]
+ all_workdirs = [f'{shell_quote(name)} {shell_quote(path)}' for name, path in workdirs]
with open('accelerator.conf', 'w') as fh:
fh.write(config_template.format(
name=shell_quote(options.name),
diff --git a/accelerator/shell/job.py b/accelerator/shell/job.py
index 2f8084ba..d9a8c886 100644
--- a/accelerator/shell/job.py
+++ b/accelerator/shell/job.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
from traceback import print_exc
from datetime import datetime
import errno
@@ -50,16 +46,16 @@ def show(url, job, verbose, show_output):
print(encode_setup(setup, as_str=True))
else:
starttime = datetime.fromtimestamp(setup.starttime).replace(microsecond=0)
- hdr = '%s (%s) at %s' % (job, job.method, starttime,)
+ hdr = f'{job} ({job.method}) at {starttime}'
if 'exectime' in setup:
- hdr = '%s in %s' % (hdr, fmttime(setup.exectime.total),)
+ hdr = f'{hdr} in {fmttime(setup.exectime.total)}'
print(colour(hdr, 'job/header'))
if job.is_build:
print(colour(' build job', 'job/highlight'))
if job.parent:
- built_from = " built from %s (%s)" % (job.parent, job.parent.method,)
+ built_from = f" built from {job.parent} ({job.parent.method})"
if job.build_job and job.parent != job.build_job:
- built_from = "%s, build job %s (%s)" % (built_from, job.build_job, job.build_job.method,)
+ built_from = f"{built_from}, build job {job.build_job} ({job.build_job.method})"
print(colour(built_from, 'job/highlight'))
things = []
def opt_thing(name):
@@ -75,17 +71,17 @@ def opt_thing(name):
opt_thing('datasets')
opt_thing('jobs')
for k, v in things:
- print(' %s: %s' % (k, v,))
+ print(f' {k}: {v}')
def list_of_things(name, things):
total = len(things)
if total > 5 and not verbose:
things = things[:3]
print()
- print(colour('%s:' % (name,), 'job/header'))
+ print(colour(f'{name}:', 'job/header'))
for thing in things:
print(' ', thing)
if total > len(things):
- print(' ... and %d more' % (total - len(things),))
+ print(f' ... and {total - len(things)} more')
if job.datasets:
list_of_things('datasets', [ds.quoted for ds in job.datasets])
try:
@@ -120,7 +116,7 @@ def list_of_things(name, things):
print(job, 'produced no output')
print()
elif out:
- print('%s produced %d bytes of output, use --output/-o to see it' % (job, sum(len(v) for v in out.values()),))
+ print(f'{job} produced {sum((len(v) for v in out.values()))} bytes of output, use --output/-o to see it')
print()
def show_source(job, pattern='*'):
@@ -131,7 +127,7 @@ def show_source(job, pattern='*'):
members = [info for info in all_members if fnmatch(info.path, pattern)]
if not members:
if pattern:
- print(colour('No sources matching %r in %s.' % (pattern, job,), 'job/warning'), file=sys.stderr)
+ print(colour(f'No sources matching {pattern!r} in {job}.', 'job/warning'), file=sys.stderr)
fh = sys.stderr
res = 1
else:
@@ -167,7 +163,7 @@ def show_file(job, pattern):
if not files:
if pattern:
fh = sys.stderr
- print(colour('No files matching %r in %s.' % (pattern, job,), 'job/warning'), file=fh)
+ print(colour(f'No files matching {pattern!r} in {job}.', 'job/warning'), file=fh)
res = 1
else:
fh = sys.stdout
@@ -206,7 +202,7 @@ def show_output_d(d, verbose):
else:
print()
if isinstance(k, int):
- k = 'analysis(%d)' % (k,)
+ k = f'analysis({k})'
print(colour(k, 'job/header'))
print(colour('=' * len(k), 'job/header'))
print(out, end='' if out.endswith('\n') else '\n')
@@ -265,6 +261,6 @@ def main(argv, cfg):
if isinstance(e, OSError) and e.errno == errno.EPIPE:
raise
print_exc(file=sys.stderr)
- print("Failed to show %r" % (path,), file=sys.stderr)
+ print(f"Failed to show {path!r}", file=sys.stderr)
res = 1
return res
diff --git a/accelerator/shell/lined.py b/accelerator/shell/lined.py
index 84966ee9..02849f92 100644
--- a/accelerator/shell/lined.py
+++ b/accelerator/shell/lined.py
@@ -17,15 +17,12 @@
# #
############################################################################
-from __future__ import division, print_function
-
from itertools import cycle
import errno
import os
import sys
from accelerator.colourwrapper import colour
-from accelerator.compat import PY2
from accelerator import mp
@@ -46,7 +43,7 @@ def split_colour(spec):
elif 40 <= code <= 48 or 100 <= code <= 107:
target = bg
elif code not in (39, 49):
- print("Sorry, %s can only use colours, not attributes" % (spec,), file=sys.stderr)
+ print(f"Sorry, {spec} can only use colours, not attributes", file=sys.stderr)
sys.exit(1)
target.append(part)
return ';'.join(fg), ';'.join(bg)
@@ -99,7 +96,7 @@ def close(self):
os.close(self.saved_stdout)
self.process.join()
if self.process.exitcode and self.process.exitcode != _RC_EPIPE:
- raise Exception('Liner process exited with %s' % (self.process.exitcode,))
+ raise Exception(f'Liner process exited with {self.process.exitcode}')
def enable_lines(colour_prefix, lined=True, decode_lines=False, max_count=None, after=0):
@@ -128,12 +125,8 @@ def lineme(lined, max_count, after):
(pre_fg1, pre_bg1),
])
- if PY2:
- in_fh = sys.stdin
- errors = 'replace'
- else:
- in_fh = sys.stdin.buffer.raw
- errors = 'surrogateescape'
+ in_fh = sys.stdin.buffer.raw
+ errors = 'surrogateescape'
if decode_lines:
if lined:
@@ -176,11 +169,11 @@ def decode_part(part):
todo = iter(line)
data = []
if line_fg and line_bg:
- data.append('\x1b[%s;%sm' % (line_fg, line_bg,))
+ data.append(f'\x1b[{line_fg};{line_bg}m')
elif line_bg:
- data.append('\x1b[%sm' % (line_bg,))
+ data.append(f'\x1b[{line_bg}m')
elif line_fg:
- data.append('\x1b[%sm' % (line_fg,))
+ data.append(f'\x1b[{line_fg}m')
if line_bg and not decode_lines:
data.append('\x1b[K') # try to fill the line with bg (if terminal does BCE)
for c in todo:
diff --git a/accelerator/shell/method.py b/accelerator/shell/method.py
index 7e09dae4..f5a0728d 100644
--- a/accelerator/shell/method.py
+++ b/accelerator/shell/method.py
@@ -18,10 +18,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
from accelerator.compat import terminal_size
from accelerator.unixhttp import call
from accelerator.shell import printdesc
@@ -38,7 +34,7 @@ def main(argv, cfg):
for name in args.method:
if name in methods:
data = methods[name]
- print('%s.%s:' % (data.package, name,))
+ print(f'{data.package}.{name}:')
if data.description.text:
for line in data.description.text.split('\n'):
if line:
@@ -47,11 +43,11 @@ def main(argv, cfg):
print()
print()
if cfg.get('interpreters'):
- print('Runs on <%s> %s' % (data.version, data.description.interpreter,))
+ print(f'Runs on <{data.version}> {data.description.interpreter}')
print()
for k in ('datasets', 'jobs',):
if data.description.get(k):
- print('%s:' % (k,))
+ print(f'{k}:')
klen = max(len(k) for k in data.description[k])
template = ' %%-%ds # %%s' % (klen,)
for k, v in data.description[k].items():
@@ -84,13 +80,13 @@ def main(argv, cfg):
else:
print(first)
else:
- print('Method %r not found' % (name,))
+ print(f'Method {name!r} not found')
else:
by_package = defaultdict(list)
for name, data in sorted(methods.items()):
by_package[data.package].append(name)
by_package.pop('accelerator.test_methods', None)
for package, names in sorted(by_package.items()):
- print('%s:' % (package,))
+ print(f'{package}:')
items = [(name, methods[name].description.text) for name in names]
printdesc(items, columns, 'method')
diff --git a/accelerator/shell/parser.py b/accelerator/shell/parser.py
index a78fdf00..c09c43fe 100644
--- a/accelerator/shell/parser.py
+++ b/accelerator/shell/parser.py
@@ -22,8 +22,6 @@
# parsing of "job specs", including as part of a dataset name.
# handles jobids, paths and method names.
-from __future__ import division, print_function
-
import argparse
import sys
from os.path import join, exists, realpath, split
@@ -35,7 +33,7 @@
from accelerator.job import Job
from accelerator.error import NoSuchJobError, NoSuchDatasetError, NoSuchWorkdirError, UrdError
from accelerator.unixhttp import call
-from accelerator.compat import url_quote, urlencode, PY3
+from accelerator.compat import url_quote, urlencode
class JobNotFound(NoSuchJobError):
pass
@@ -73,7 +71,7 @@ def split_tildes(n, allow_empty=False, extended=False):
return n, lst
def method2job(cfg, method, **kw):
- url ='%s/method2job/%s?%s' % (cfg.url, url_quote(method), urlencode(kw))
+ url =f'{cfg.url}/method2job/{url_quote(method)}?{urlencode(kw)}'
found = call(url)
if 'error' in found:
raise JobNotFound(found.error)
@@ -89,7 +87,7 @@ def job_up(job, count):
if prev:
prev = prev.job
if not prev:
- raise JobNotFound('Tried to go %d up from %s, but only %d previous jobs available' % (count, err_job, ix,))
+ raise JobNotFound(f'Tried to go {count} up from {err_job}, but only {ix} previous jobs available')
job = prev
return job
@@ -146,30 +144,30 @@ def split(n, what):
if n in ('jobs', 'datasets'):
k = n
if current or tildes:
- raise JobNotFound("Don't use !~+<>^ on .%s, put after .%s.foo(HERE)." % (k, k))
+ raise JobNotFound(f"Don't use !~+<>^ on .{k}, put after .{k}.foo(HERE).")
try:
n = next(dotted)
except StopIteration:
- raise JobNotFound("%s.%s.what?" % (job, k,))
+ raise JobNotFound(f"{job}.{k}.what?")
n, current, tildes = split(n, k)
elif n in p.jobs and n in p.datasets:
- raise JobNotFound("Job %s (%s) has %s in both .jobs and .datasets, please specify." % (job, job.method, n,))
+ raise JobNotFound(f"Job {job} ({job.method}) has {n} in both .jobs and .datasets, please specify.")
if k:
if n not in p[k]:
- raise JobNotFound("Job %s (%s) does not have a %r." % (job, job.method, k + '.' + n,))
+ raise JobNotFound(f"Job {job} ({job.method}) does not have a {k + '.' + n!r}.")
else:
if n in p.jobs:
k = 'jobs'
elif n in p.datasets:
k = 'datasets'
else:
- raise JobNotFound("Job %s (%s) does not have a %r." % (job, job.method, n,))
+ raise JobNotFound(f"Job {job} ({job.method}) does not have a {n!r}.")
if not p[k][n]:
- raise JobNotFound("%s.%s.%s is None" % (job, k, n,))
+ raise JobNotFound(f"{job}.{k}.{n} is None")
job = p[k][n]
if isinstance(job, list):
if len(job) != 1:
- raise JobNotFound("Job %s (%s) has %d %s in %r." % (job, job.method, len(job), k, n,))
+ raise JobNotFound(f"Job {job} ({job.method}) has {len(job)} {k} in {n!r}.")
job = job[0]
if isinstance(job, Dataset):
ds = job
@@ -193,14 +191,14 @@ def _name2job_do_tildes(cfg, job, current, tildes):
job = job_up(job, count)
elif char == '<':
if count > job.number:
- raise JobNotFound('Tried to go %d jobs back from %s.' % (count, job,))
+ raise JobNotFound(f'Tried to go {count} jobs back from {job}.')
job = Job._create(job.workdir, job.number - count)
elif char == '>':
job = Job._create(job.workdir, job.number + count)
else:
- raise Exception("BUG: split_tildes should not give %r as a char" % (char,))
+ raise Exception(f"BUG: split_tildes should not give {char!r} as a char")
if not exists(job.filename('setup.json')):
- raise JobNotFound('Job resolved to %r but that job does not exist' % (job,))
+ raise JobNotFound(f'Job resolved to {job!r} but that job does not exist')
return job
def _name2job(cfg, n, current):
@@ -228,12 +226,12 @@ def _name2job(cfg, n, current):
print(e, file=sys.stderr)
urdres = None
if not urdres:
- raise JobNotFound('urd list %r not found' % (a[0],))
+ raise JobNotFound(f'urd list {a[0]!r} not found')
from accelerator.build import JobList
joblist = JobList(Job(e[1], e[0]) for e in urdres.joblist)
res = joblist.get(entry)
if not res:
- raise JobNotFound('%r not found in %s' % (entry, path,))
+ raise JobNotFound(f'{entry!r} not found in {path}')
return res
if re.match(r'[^/]+-\d+$', n):
# Looks like a jobid
@@ -243,12 +241,12 @@ def _name2job(cfg, n, current):
# Looks like workdir-LATEST
wd = m.group(1)
if wd not in WORKDIRS:
- raise NoSuchWorkdirError('Not a valid workdir: "%s"' % (wd,))
+ raise NoSuchWorkdirError(f'Not a valid workdir: "{wd}"')
path = join(WORKDIRS[wd], n)
try:
n = readlink(path)
except OSError as e:
- raise JobNotFound('Failed to read %s: %s' % (path, e,))
+ raise JobNotFound(f'Failed to read {path}: {e}')
return Job(n)
if n not in ('.', '..') and '/' not in n:
# Must be a method then
@@ -258,10 +256,10 @@ def _name2job(cfg, n, current):
path, jid = split(realpath(n))
job = Job(jid)
if WORKDIRS.get(job.workdir, path) != path:
- print("### Overriding workdir %s to %s" % (job.workdir, path,))
+ print(f"### Overriding workdir {job.workdir} to {path}")
WORKDIRS[job.workdir] = path
return job
- raise JobNotFound("Don't know what to do with %r." % (n,))
+ raise JobNotFound(f"Don't know what to do with {n!r}.")
def split_ds_dir(n):
"""try to split a path at the jid/ds boundary"""
@@ -278,7 +276,7 @@ def split_ds_dir(n):
n, bit = n.rsplit('/', 1)
name_bits.append(bit)
if not n:
- raise JobNotFound('No setup.json found in %r' % (orig_n,))
+ raise JobNotFound(f'No setup.json found in {orig_n!r}')
if not name_bits:
name_bits = ['default']
return n, '/'.join(reversed(name_bits))
@@ -309,7 +307,7 @@ def follow(key, motion):
res = ds
for done in range(count):
if not getattr(res, key):
- raise DatasetNotFound('Tried to go %d %s from %s, but only %d available (stopped on %s)' % (count, motion, ds, done, res,))
+ raise DatasetNotFound(f'Tried to go {count} {motion} from {ds}, but only {done} available (stopped on {res})')
res = getattr(res, key)
return res
for char, count in tildes:
@@ -320,7 +318,7 @@ def follow(key, motion):
slices = ds.job.params.slices
from accelerator import g
if hasattr(g, 'slices'):
- assert g.slices == slices, "Dataset %s needs %d slices, by we are already using %d slices" % (ds, slices, g.slices)
+ assert g.slices == slices, f"Dataset {ds} needs {slices} slices, by we are already using {g.slices} slices"
else:
g.slices = slices
return ds
@@ -330,10 +328,9 @@ class ArgumentParser(argparse.ArgumentParser):
def __init__(self, *a, **kw):
kw = dict(kw)
kw['prefix_chars'] = '-+'
- if PY3:
- # allow_abbrev is 3.5+. it's not even available in the pypi backport of argparse.
- # it also regrettably disables -abc for -a -b -c until 3.8.
- kw['allow_abbrev'] = False
+ # allow_abbrev is 3.5+. it's not even available in the pypi backport of argparse.
+ # it also regrettably disables -abc for -a -b -c until 3.8.
+ kw['allow_abbrev'] = False
return argparse.ArgumentParser.__init__(self, *a, **kw)
def add_argument(self, *a, **kw):
diff --git a/accelerator/shell/script.py b/accelerator/shell/script.py
index 5fef7a01..a9aa2013 100644
--- a/accelerator/shell/script.py
+++ b/accelerator/shell/script.py
@@ -18,10 +18,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
import sys
from glob import glob
from os.path import dirname, basename
@@ -61,7 +57,7 @@ def main(argv, cfg):
try:
module = import_module(modname)
except Exception as e:
- print(colour('%s: %s' % (item, e,), 'script/warning'), file=sys.stderr)
+ print(colour(f'{item}: {e}', 'script/warning'), file=sys.stderr)
continue
scripts.append((name, getattr(module, 'description', '')))
diff --git a/accelerator/shell/sherlock.py b/accelerator/shell/sherlock.py
index 21ba5812..6ad9ed87 100644
--- a/accelerator/shell/sherlock.py
+++ b/accelerator/shell/sherlock.py
@@ -17,8 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-
from datetime import datetime
import os
import sys
@@ -39,7 +37,7 @@ def validate(data):
job = None
if job:
if job.path != data.job:
- warnings.append("path mismatch (%r != %r)" % (job.path, data.job,))
+ warnings.append(f"path mismatch ({job.path!r} != {data.job!r})")
else:
warnings.append("unknown job")
h = b64hash_setup(data.job + '/setup.json')
@@ -87,14 +85,14 @@ def main(argv, cfg):
print(prefix + colour('decoding error', 'sherlock/warning'), file=sys.stderr)
continue
if args.verbose:
- print('%s%s' % (prefix, data.job,), end='')
+ print(f'{prefix}{data.job}', end='')
else:
- print('%s%s' % (prefix, data.job.rsplit('/', 1)[-1],), end='')
+ print(f"{prefix}{data.job.rsplit('/', 1)[-1]}", end='')
warnings = validate(data)
if warnings:
print(' ' + colour(', '.join(warnings), 'sherlock/warning'), end='')
if args.verbose:
ts = datetime.fromtimestamp(data.time)
- print(' (%s at %s on %s)' % (data.method, ts, data.host), end='')
+ print(f' ({data.method} at {ts} on {data.host})', end='')
print()
return res
diff --git a/accelerator/shell/status.py b/accelerator/shell/status.py
index fffc57ea..6a6d9365 100644
--- a/accelerator/shell/status.py
+++ b/accelerator/shell/status.py
@@ -17,8 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-
from accelerator.build import fmttime
from accelerator.error import JobError
from accelerator.job import Job
@@ -40,14 +38,13 @@ def main(argv, cfg):
error = call(cfg.url + '/last_error')
t = datetime.fromtimestamp(error.time).replace(microsecond=0)
print()
- print('Last error at %s:' % (t,))
+ print(f'Last error at {t}:')
for jobid, method, status in error.last_error:
e = JobError(Job(jobid, method), method, status)
print(e.format_msg(), file=sys.stderr)
else:
if args.short:
t = fmttime(status.report_t - status.current[0], True)
- print('%s (%s)' % (status.current[1], t))
+ print(f'{status.current[1]} ({t})')
else:
print_status_stacks(status.status_stacks)
-
diff --git a/accelerator/shell/urd.py b/accelerator/shell/urd.py
index 5e34d50d..51edfb4d 100644
--- a/accelerator/shell/urd.py
+++ b/accelerator/shell/urd.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
import sys
from os import environ
from argparse import RawDescriptionHelpFormatter
@@ -91,7 +87,7 @@ def urd_get(path):
if path.startswith(':'):
a = path[1:].split(':', 1)
if len(a) == 1:
- print('%r should have two or no :' % (path,), file=sys.stderr)
+ print(f'{path!r} should have two or no :', file=sys.stderr)
return None, None
path = a[0]
try:
@@ -103,10 +99,10 @@ def urd_get(path):
entry = tildes = None
path = resolve_path_part(path)
if len(path) != 3 and tildes:
- print("path %r isn't walkable (~^)" % ('/'.join(path),), file=sys.stderr)
+ print(f"path {'/'.join(path)!r} isn't walkable (~^)", file=sys.stderr)
return None, None
if len(path) != 3 and entry is not None:
- print("path %r doesn't take an entry (%r)" % ('/'.join(path), entry,), file=sys.stderr)
+ print(f"path {'/'.join(path)!r} doesn't take an entry ({entry!r})", file=sys.stderr)
return None, None
try:
res = urd_call_w_tildes(cfg, '/'.join(path), tildes)
@@ -143,7 +139,7 @@ def fmt_caption(path, caption, indent):
return joblist.get(entry, '')
if res['deps']:
deps = sorted(
- ('%s/%s' % (k, v['timestamp'],), v['caption'],)
+ (f"{k}/{v['timestamp']}", v['caption'],)
for k, v in res['deps'].items()
)
if len(deps) > 1:
diff --git a/accelerator/shell/workdir.py b/accelerator/shell/workdir.py
index 494e5532..dc1f8a93 100644
--- a/accelerator/shell/workdir.py
+++ b/accelerator/shell/workdir.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
import sys
import os
@@ -76,7 +72,7 @@ def workdir_jids(cfg, name):
if wd == name and num.isdigit():
jidlist.append(int(num))
jidlist.sort()
- return ['%s-%s' % (name, jid,) for jid in jidlist]
+ return [f'{name}-{jid}' for jid in jidlist]
def main(argv, cfg):
usage = "%(prog)s [-p] [-a | [workdir [workdir [...]]]"
diff --git a/accelerator/standard_methods/a_csvexport.py b/accelerator/standard_methods/a_csvexport.py
index 79d9ffeb..a68dafca 100644
--- a/accelerator/standard_methods/a_csvexport.py
+++ b/accelerator/standard_methods/a_csvexport.py
@@ -18,9 +18,6 @@
# #
############################################################################
-from __future__ import division
-from __future__ import absolute_import
-
description = r'''Dataset (or chain) to CSV file.'''
from shutil import copyfileobj
@@ -32,7 +29,7 @@
from itertools import chain
import gzip
-from accelerator.compat import PY3, PY2, izip, imap, long
+from accelerator.compat import izip, imap
from accelerator import status
@@ -71,16 +68,10 @@ def wrapped_write(s):
json=JSONEncoder(sort_keys=True, ensure_ascii=True, check_circular=False).encode,
)
-if PY3:
- enc = str
- format['bytes'] = lambda s: s.decode('utf-8', errors='backslashreplace')
- format['number'] = repr
- format['unicode'] = None
-else:
- enc = lambda s: s.encode('utf-8')
- format['bytes'] = None
- format['number'] = lambda n: str(n) if isinstance(n, long) else repr(n)
- format['unicode'] = lambda s: s.encode('utf-8')
+enc = str
+format['bytes'] = lambda s: s.decode('utf-8', errors='backslashreplace')
+format['number'] = repr
+format['unicode'] = None
def csvexport(sliceno, filename, labelsonfirstline):
d = datasets.source[0]
@@ -100,14 +91,11 @@ def csvexport(sliceno, filename, labelsonfirstline):
open_func = partial(gzip.open, compresslevel=options.compression)
else:
open_func = open
- if PY2:
- open_func = partial(open_func, mode='wb')
- else:
- open_func = partial(open_func, mode='xt', encoding='utf-8')
+ open_func = partial(open_func, mode='xt', encoding='utf-8')
if options.none_as:
if isinstance(options.none_as, dict):
bad_none = set(options.none_as) - set(options.labels)
- assert not bad_none, 'Unknown labels in none_as: %r' % (bad_none,)
+ assert not bad_none, f'Unknown labels in none_as: {bad_none!r}'
else:
assert isinstance(options.none_as, str), "What did you pass as none_as?"
def resolve_none(label, col):
@@ -207,7 +195,7 @@ def analysis(sliceno, job):
if '%' in options.filename:
filename = options.filename % (sliceno,)
else:
- filename = '%s.%d' % (options.filename, sliceno,)
+ filename = f'{options.filename}.{sliceno}'
csvexport(sliceno, filename, options.labelsonfirstline)
job.register_file(filename)
else:
@@ -217,7 +205,7 @@ def analysis(sliceno, job):
def synthesis(job, slices):
if not options.sliced:
def msg(sliceno):
- return "Assembling %s (%d/%d)" % (options.filename, sliceno + 1, slices,)
+ return f"Assembling {options.filename} ({sliceno + 1}/{slices})"
with status(msg(0)) as update:
with job.open(options.filename, "wb") as outfh:
for sliceno in range(slices):
diff --git a/accelerator/standard_methods/a_csvimport.py b/accelerator/standard_methods/a_csvimport.py
index a77228a3..b67e75ed 100644
--- a/accelerator/standard_methods/a_csvimport.py
+++ b/accelerator/standard_methods/a_csvimport.py
@@ -19,10 +19,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
CSV file to dataset.
@@ -92,7 +88,7 @@ def reader_status(status_fd, update):
pass
count = 0
while True:
- update('{0:n} lines read'.format(count))
+ update(f'{count:n} lines read')
data = os.read(status_fd, 8)
if not data:
break
@@ -126,7 +122,7 @@ def char2int(name, empty_value, specials="empty"):
char = options.get(name)
if not char:
return empty_value
- msg = "%s must be a single iso-8859-1 character (or %s)" % (name, specials,)
+ msg = f"{name} must be a single iso-8859-1 character (or {specials})"
if isinstance(char, bytes):
char = uni(char)
try:
@@ -137,7 +133,7 @@ def char2int(name, empty_value, specials="empty"):
return cstuff.backend.char2int(char)
def import_slice(fallback_msg, fd, sliceno, slices, field_count, out_fns, gzip_mode, separator, r_num, quote_char, lf_char, allow_bad, allow_extra_empty):
- fn = "import.success.%d" % (sliceno,)
+ fn = f"import.success.{sliceno}"
fh = open(fn, "wb+")
real_stderr = os.dup(2)
try:
@@ -230,7 +226,7 @@ def prepare(job, slices):
if options.strip_labels:
labels = [x.strip() for x in labels]
labels = [options.rename.get(x, x) for x in labels]
- assert len(labels) == len(set(labels)), "Duplicate labels: %r" % (labels,)
+ assert len(labels) == len(set(labels)), f"Duplicate labels: {labels!r}"
dw = job.datasetwriter(
columns={n: 'bytes' for n in labels if n not in options.discard},
@@ -314,7 +310,7 @@ def analysis(sliceno, slices, prepare_res, update_top_status):
r_num = cstuff.mk_uint64(9) # [good_count, bad_count, comment_count, good_min,max, bad_min,max, comment_min,max]
gzip_mode = b"wb%d" % (options.compression,)
try:
- import_slice("c backend failed in slice %d" % (sliceno,), fds[sliceno], sliceno, slices, len(labels), out_fns, gzip_mode, separator, r_num, quote_char, lf_char, options.allow_bad, options.allow_extra_empty)
+ import_slice(f"c backend failed in slice {sliceno}", fds[sliceno], sliceno, slices, len(labels), out_fns, gzip_mode, separator, r_num, quote_char, lf_char, options.allow_bad, options.allow_extra_empty)
finally:
os.close(fds[sliceno])
return list(r_num)
diff --git a/accelerator/standard_methods/a_csvimport_zip.py b/accelerator/standard_methods/a_csvimport_zip.py
index 79e02b82..ee3cb5c4 100644
--- a/accelerator/standard_methods/a_csvimport_zip.py
+++ b/accelerator/standard_methods/a_csvimport_zip.py
@@ -18,11 +18,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-from __future__ import absolute_import
-
description = r'''
Call csvimport on one or more files in a zip file.
@@ -125,7 +120,7 @@ def tmpfn():
used_names.add(name)
res.append((next(tmpfn), info, name, fn,))
if namemap:
- raise Exception("The following files were not found in %s: %r" % (options.filename, set(namemap),))
+ raise Exception(f"The following files were not found in {options.filename}: {set(namemap)!r}")
if options.chaining == 'by_filename':
res.sort(key=lambda x: x[3])
if options.chaining == 'by_dsname':
@@ -152,7 +147,7 @@ def step(self, msg):
self.z_so_far += self.z[self.cnt_so_far]
self.cnt_so_far += 1
percent = self.z_so_far / self.z_total * 100
- return '%s %s (file %d/%d, up to %d%% of total size)' % (msg, fn, self.cnt_so_far, self.cnt_total, percent,)
+ return f'{msg} {fn} (file {self.cnt_so_far}/{self.cnt_total}, up to {percent}% of total size)'
def analysis(sliceno, slices, prepare_res, job):
lst = prepare_res[sliceno::slices]
@@ -174,7 +169,7 @@ def synthesis(prepare_res):
for fn, info, dsn in lst:
update(msg.step('importing'))
opts.filename = fn
- show_fn = '%s:%s' % (options.filename, info.filename,)
+ show_fn = f'{options.filename}:{info.filename}'
ds = build('csvimport', options=opts, previous=previous, caption='Import of ' + show_fn).dataset()
previous = ds.link_to_here(dsn, filename=show_fn)
if options.chaining == 'off':
diff --git a/accelerator/standard_methods/a_dataset_checksum.py b/accelerator/standard_methods/a_dataset_checksum.py
index 27b0c75f..2997c995 100644
--- a/accelerator/standard_methods/a_dataset_checksum.py
+++ b/accelerator/standard_methods/a_dataset_checksum.py
@@ -18,10 +18,6 @@
# #
############################################################################
-from __future__ import division
-from __future__ import print_function
-from __future__ import absolute_import
-
description = r'''
Take a dataset and make a checksum of one or more columns.
@@ -43,7 +39,6 @@
from heapq import merge
from accelerator.extras import DotDict
-from accelerator.compat import PY2
options = dict(
columns = set(),
@@ -52,11 +47,8 @@
datasets = ('source',)
-if PY2:
- bytesrepr = repr
-else:
- def bytesrepr(v):
- return repr(v).encode('utf-8')
+def bytesrepr(v):
+ return repr(v).encode('utf-8')
def bytesstr(v):
return v.encode('utf-8')
@@ -86,7 +78,7 @@ def prepare():
# Same with pickle, but worse (many picklable values will break this).
for n in columns:
col = datasets.source.columns[n]
- if col.type == 'bytes' or (col.type == 'ascii' and PY2):
+ if col.type == 'bytes':
# doesn't need any encoding, but might need None-handling.
if col.none_support:
translators[n] = self_none
@@ -99,7 +91,7 @@ def prepare():
translators[n] = sortdicts
elif col.type == 'pickle':
translators[n] = sortdicts
- print('WARNING: Column %s is pickle, may not work' % (n,))
+ print(f'WARNING: Column {n} is pickle, may not work')
else:
translators[n] = bytesrepr
return columns, translators
@@ -124,5 +116,5 @@ def synthesis(prepare_res, analysis_res):
else:
all = chain.from_iterable(analysis_res)
res = md5(b''.join(all)).hexdigest()
- print("%s: %s" % (datasets.source, res,))
+ print(f"{datasets.source}: {res}")
return DotDict(sum=int(res, 16), sort=options.sort, columns=prepare_res[0], source=datasets.source)
diff --git a/accelerator/standard_methods/a_dataset_checksum_chain.py b/accelerator/standard_methods/a_dataset_checksum_chain.py
index 2488d055..07473191 100644
--- a/accelerator/standard_methods/a_dataset_checksum_chain.py
+++ b/accelerator/standard_methods/a_dataset_checksum_chain.py
@@ -18,10 +18,6 @@
# #
############################################################################
-from __future__ import division
-from __future__ import print_function
-from __future__ import absolute_import
-
description = r'''
Take a chain of datasets and make a checksum of one or more columns.
See dataset_checksum.description for more information.
@@ -48,5 +44,5 @@ def synthesis():
for src in jobs:
data = build('dataset_checksum', columns=options.columns, sort=options.sort, source=src).load()
sum ^= data.sum
- print("Total: %016x" % (sum,))
+ print(f"Total: {sum:016x}")
return DotDict(sum=sum, columns=data.columns, sort=options.sort, sources=jobs)
diff --git a/accelerator/standard_methods/a_dataset_concat.py b/accelerator/standard_methods/a_dataset_concat.py
index e378b661..111b688a 100644
--- a/accelerator/standard_methods/a_dataset_concat.py
+++ b/accelerator/standard_methods/a_dataset_concat.py
@@ -35,9 +35,9 @@ def prepare(job):
hashlabel = datasets.source.hashlabel
for ds in chain:
if columns != {name: col.type for name, col in ds.columns.items()}:
- raise Exception('Dataset %s does not have the same columns as %s' % (ds.quoted, datasets.source.quoted,))
+ raise Exception(f'Dataset {ds.quoted} does not have the same columns as {datasets.source.quoted}')
if hashlabel != ds.hashlabel:
- raise Exception('Dataset %s has hashlabel %r, expected %r' % (ds.quoted, ds.hashlabel, hashlabel,))
+ raise Exception(f'Dataset {ds.quoted} has hashlabel {ds.hashlabel!r}, expected {hashlabel!r}')
dw = job.datasetwriter(hashlabel=hashlabel, previous=datasets.previous, copy_mode=True)
for name, t in sorted(columns.items()):
dw.add(name, t, none_support=chain.none_support(name))
diff --git a/accelerator/standard_methods/a_dataset_fanout.py b/accelerator/standard_methods/a_dataset_fanout.py
index f7dd8e6a..97f83246 100644
--- a/accelerator/standard_methods/a_dataset_fanout.py
+++ b/accelerator/standard_methods/a_dataset_fanout.py
@@ -29,7 +29,7 @@
import itertools
import re
-from accelerator.compat import unicode, izip
+from accelerator.compat import izip
from accelerator import OptionString, NoSuchDatasetError
from accelerator import subjobs, status
@@ -52,7 +52,7 @@ def prepare(job):
seen_all = set(chain[0].columns)
for ds in chain:
if options.column not in ds.columns:
- raise Exception('%r does not have column %r' % (ds, options.column,))
+ raise Exception(f'{ds!r} does not have column {options.column!r}')
hashlabel.add(ds.hashlabel)
seen_all &= set(ds.columns)
for name, col in ds.columns.items():
@@ -60,7 +60,7 @@ def prepare(job):
none_support[name] |= col.none_support
seen_all.discard(options.column)
if not seen_all:
- raise Exception('Chain has no common columns (except %r)' % (options.column,))
+ raise Exception(f'Chain has no common columns (except {options.column!r})')
columns = {k: columns[k] for k in seen_all}
if len(hashlabel) == 1:
hashlabel = hashlabel.pop()
@@ -86,7 +86,7 @@ def prepare(job):
if len(types) > 1 and not (types - {'int32', 'int64', 'float32', 'float64'}):
types = {'number'}
if len(types) > 1:
- raise Exception("Column %r has incompatible types: %r" % (name, types,))
+ raise Exception(f"Column {name!r} has incompatible types: {types!r}")
columns[name] = (types.pop(), none_support[name],)
collect = subjobs.build(
@@ -103,7 +103,7 @@ def prepare(job):
else:
previous = {}
- with status('Creating %d datasets' % (len(values),)):
+ with status(f'Creating {len(values)} datasets'):
writers = {
name: job.datasetwriter(
name=name,
@@ -123,4 +123,4 @@ def analysis(sliceno, prepare_res):
# we can't just use chain.iterate because of protections against changing types with copy_mode
values_it = itertools.chain.from_iterable(ds.iterate(sliceno, columns, copy_mode=True, status_reporting=False) for ds in chain)
for key, values in izip(key_it, values_it):
- writers[unicode(key)].write(*values)
+ writers[str(key)].write(*values)
diff --git a/accelerator/standard_methods/a_dataset_fanout_collect.py b/accelerator/standard_methods/a_dataset_fanout_collect.py
index 8cb4e00d..db10b041 100644
--- a/accelerator/standard_methods/a_dataset_fanout_collect.py
+++ b/accelerator/standard_methods/a_dataset_fanout_collect.py
@@ -24,7 +24,7 @@
'''
from accelerator import OptionString
-from accelerator.compat import unicode, imap
+from accelerator.compat import imap
options = {
@@ -39,7 +39,7 @@
def analysis(sliceno):
chain = datasets.source.chain(stop_ds={jobs.previous: 'source'}, length=options.length)
- return set(imap(unicode, chain.iterate(sliceno, options.column)))
+ return set(imap(str, chain.iterate(sliceno, options.column)))
def synthesis(analysis_res):
return analysis_res.merge_auto()
diff --git a/accelerator/standard_methods/a_dataset_filter_columns.py b/accelerator/standard_methods/a_dataset_filter_columns.py
index 7d136084..a1d43ed6 100644
--- a/accelerator/standard_methods/a_dataset_filter_columns.py
+++ b/accelerator/standard_methods/a_dataset_filter_columns.py
@@ -19,8 +19,6 @@
# #
############################################################################
-from __future__ import absolute_import
-
description = r"""Make only some columns from a dataset visible."""
from accelerator.extras import OptionDefault
diff --git a/accelerator/standard_methods/a_dataset_hashpart.py b/accelerator/standard_methods/a_dataset_hashpart.py
index 67132c8c..8654b674 100644
--- a/accelerator/standard_methods/a_dataset_hashpart.py
+++ b/accelerator/standard_methods/a_dataset_hashpart.py
@@ -18,9 +18,6 @@
# #
############################################################################
-from __future__ import division
-from __future__ import absolute_import
-
description = r'''
Rewrite a dataset (or chain to previous) with new hashlabel.
'''
@@ -58,9 +55,9 @@ def prepare_one(ix, source, previous, job, chain, slices):
if sliceno == slices - 1 and options.chain_slices and ix == len(chain) - 1:
name = "default"
else:
- name = '%d.%d' % (ix, sliceno,)
+ name = f'{ix}.{sliceno}'
dw = job.datasetwriter(
- caption="%s (slice %d)" % (caption, sliceno),
+ caption=f"{caption} (slice {sliceno})",
hashlabel=options.hashlabel,
filename=filename,
previous=previous,
diff --git a/accelerator/standard_methods/a_dataset_sort.py b/accelerator/standard_methods/a_dataset_sort.py
index 5e85f2f0..441c9e7e 100644
--- a/accelerator/standard_methods/a_dataset_sort.py
+++ b/accelerator/standard_methods/a_dataset_sort.py
@@ -18,10 +18,6 @@
# #
############################################################################
-from __future__ import division
-from __future__ import absolute_import
-from __future__ import print_function
-
description = r'''
Stable sort a dataset based on one or more columns.
You'll have to type the sort column(s) approprietly.
@@ -212,7 +208,7 @@ def analysis(sliceno, params, prepare_res):
sort_idx, _ = sort(partial(ds_list.iterate, sliceno))
columniter = partial(ds_list.iterate, sliceno, copy_mode=True)
for ix, column in enumerate(datasets.source.columns, 1):
- colstat = '%r (%d/%d)' % (column, ix, len(datasets.source.columns),)
+ colstat = f'{column!r} ({ix}/{len(datasets.source.columns)})'
with status('Reading ' + colstat):
lst = list(columniter(column))
with status('Writing ' + colstat):
diff --git a/accelerator/standard_methods/a_dataset_type.py b/accelerator/standard_methods/a_dataset_type.py
index bb81c33f..d693f6bc 100644
--- a/accelerator/standard_methods/a_dataset_type.py
+++ b/accelerator/standard_methods/a_dataset_type.py
@@ -18,10 +18,6 @@
# #
############################################################################
-from __future__ import division
-from __future__ import absolute_import
-from __future__ import print_function
-
from resource import getpagesize
from os import unlink
from os.path import exists
@@ -29,7 +25,7 @@
from shutil import copyfileobj
from struct import Struct
-from accelerator.compat import unicode, itervalues, PY2
+from accelerator.compat import itervalues
from accelerator.extras import OptionEnum, DotDict, quote
from accelerator.dsutil import typed_writer, typed_reader
@@ -135,7 +131,7 @@ def prepare_one(ix, source, chain, job, slices, previous_res):
for k, v in options.rename.items():
if k in source.columns:
if v in rev_rename:
- raise Exception('Both column %r and column %r rename to %r (in %s)' % (rev_rename[v], k, v, source_name))
+ raise Exception(f'Both column {rev_rename[v]!r} and column {k!r} rename to {v!r} (in {source_name})')
if v is not None:
rev_rename[v] = k
if v in column2type:
@@ -147,20 +143,20 @@ def prepare_one(ix, source, chain, job, slices, previous_res):
# renamed over don't need to be duplicated
just_rename[k] = dup_rename.pop(k)
if dup_rename:
- dup_ds = source.link_to_here(name='dup.%d' % (ix,), rename=dup_rename, column_filter=dup_rename.values())
+ dup_ds = source.link_to_here(name=f'dup.{ix}', rename=dup_rename, column_filter=dup_rename.values())
if source.hashlabel in dup_rename:
just_rename[source.hashlabel] = None
if just_rename:
- source = source.link_to_here(name='rename.%d' % (ix,), rename=just_rename)
+ source = source.link_to_here(name=f'rename.{ix}', rename=just_rename)
if dup_rename:
- source = source.merge(dup_ds, name='merge.%d' % (ix,))
+ source = source.merge(dup_ds, name=f'merge.{ix}')
none_support = set()
for colname, coltype in column2type.items():
if colname not in source.columns:
- raise Exception("Dataset %s doesn't have a column named %r (has %r)" % (source_name, colname, set(source.columns),))
+ raise Exception(f"Dataset {source_name} doesn't have a column named {colname!r} (has {set(source.columns)!r})")
dc = source.columns[colname]
if dc.type not in byteslike_types:
- raise Exception("Dataset %s column %r is type %s, must be one of %r" % (source_name, colname, dc.type, byteslike_types,))
+ raise Exception(f"Dataset {source_name} column {colname!r} is type {dc.type}, must be one of {byteslike_types!r}")
coltype = coltype.split(':', 1)[0]
if coltype.endswith('+None'):
coltype = coltype[:-5]
@@ -191,7 +187,7 @@ def prepare_one(ix, source, chain, job, slices, previous_res):
parent = source
if hashlabel and hashlabel not in columns:
if options.hashlabel:
- raise Exception("Can't rehash %s on discarded column %r." % (source_name, hashlabel,))
+ raise Exception(f"Can't rehash {source_name} on discarded column {hashlabel!r}.")
hashlabel = None # it gets inherited from the parent if we're keeping it.
hashlabel_override = False
columns = {
@@ -212,10 +208,10 @@ def prepare_one(ix, source, chain, job, slices, previous_res):
name = ds_name
else:
# This ds is either an earlier part of the chain or will be merged into ds_name in synthesis
- name = '%s.%d' % (ds_name, sliceno,)
+ name = f'{ds_name}.{sliceno}'
dw = job.datasetwriter(
columns=columns,
- caption='%s (from %s slice %d)' % (options.caption, source_name, sliceno,),
+ caption=f'{options.caption} (from {source_name} slice {sliceno})',
hashlabel=hashlabel,
filename=filename,
previous=previous,
@@ -231,7 +227,7 @@ def prepare_one(ix, source, chain, job, slices, previous_res):
dw = job.datasetwriter(
name=ds_name,
columns=columns,
- caption='%s (from %s)' % (options.caption, source_name,),
+ caption=f'{options.caption} (from {source_name})',
hashlabel=hashlabel,
hashlabel_override=hashlabel_override,
filename=filename,
@@ -293,7 +289,7 @@ def analysis(sliceno, slices, prepare_res):
dataset_type.numeric_comma = True
res = [analysis_one(sliceno, slices, p) for p in prepare_res]
for fn in ('slicemap', 'badmap',):
- fn = '%s%d' % (fn, sliceno,)
+ fn = f'{fn}{sliceno}'
if exists(fn):
unlink(fn)
return res
@@ -331,7 +327,7 @@ def analysis_one(sliceno, slices, prepare_res):
column2type=column2type,
)
if options.filter_bad:
- vars.badmap_fd = map_init(vars, 'badmap%d' % (sliceno,))
+ vars.badmap_fd = map_init(vars, f'badmap{sliceno}')
bad_count, default_count, minmax = analysis_lap(vars)
if sum(sum(c) for c in itervalues(bad_count)):
vars.first_lap = False
@@ -373,28 +369,23 @@ def __getitem__(self, key):
def __setitem__(self, key, value):
self._s.pack_into(self.inner, key * 2, value)
def __iter__(self):
- if PY2:
- def it():
- for o in range(len(self.inner) // 2):
- yield self[o]
- else:
- def it():
- for v, in self._s.iter_unpack(self.inner):
- yield v
+ def it():
+ for v, in self._s.iter_unpack(self.inner):
+ yield v
return it()
def analysis_lap(vars):
if vars.rehashing:
if vars.first_lap:
- out_fn = 'hashtmp.%d' % (vars.sliceno,)
+ out_fn = f'hashtmp.{vars.sliceno}'
colname = vars.dw.hashlabel
coltype = vars.column2type[colname]
vars.rehashing = False
real_coltype = one_column(vars, colname, coltype, [out_fn], True)
vars.rehashing = True
assert vars.res_bad_count[colname] == [0] # implicitly has a default
- vars.slicemap_fd = map_init(vars, 'slicemap%d' % (vars.sliceno,), 'slicemap_size')
+ vars.slicemap_fd = map_init(vars, f'slicemap{vars.sliceno}', 'slicemap_size')
slicemap = mmap(vars.slicemap_fd, vars.slicemap_size)
vars.map_fhs.append(slicemap)
slicemap = Int16BytesWrapper(slicemap)
@@ -429,7 +420,7 @@ def one_column(vars, colname, coltype, out_fns, for_hasher=False):
else:
record_bad = 0
skip_bad = options.filter_bad
- minmax_fn = 'minmax%d' % (vars.sliceno,)
+ minmax_fn = f'minmax{vars.sliceno}'
if coltype.split(':')[0].endswith('+None'):
coltype = ''.join(coltype.split('+None', 1))
@@ -473,11 +464,11 @@ def one_column(vars, colname, coltype, out_fns, for_hasher=False):
offsets = []
max_counts = []
d = vars.source
- assert colname in d.columns, '%s not in %s' % (colname, d.quoted,)
+ assert colname in d.columns, f'{colname} not in {d.quoted}'
if not is_null_converter:
- assert d.columns[colname].type in byteslike_types, '%s has bad type in %s' % (colname, d.quoted,)
+ assert d.columns[colname].type in byteslike_types, f'{colname} has bad type in {d.quoted}'
in_fns.append(d.column_filename(colname, vars.sliceno))
- in_msgnames.append('%s column %s slice %d' % (d.quoted, quote(colname), vars.sliceno,))
+ in_msgnames.append(f'{d.quoted} column {quote(colname)} slice {vars.sliceno}')
if d.columns[colname].offsets:
offsets.append(d.columns[colname].offsets[vars.sliceno])
max_counts.append(d.lines[vars.sliceno])
@@ -495,7 +486,7 @@ def one_column(vars, colname, coltype, out_fns, for_hasher=False):
else:
default_value_is_None = False
if default_value != cstuff.NULL:
- if isinstance(default_value, unicode):
+ if isinstance(default_value, str):
default_value = default_value.encode("utf-8")
default_len = len(default_value)
c = getattr(cstuff.backend, 'convert_column_' + cfunc)
@@ -505,7 +496,7 @@ def one_column(vars, colname, coltype, out_fns, for_hasher=False):
c_slices = 1
bad_count = cstuff.mk_uint64(c_slices)
default_count = cstuff.mk_uint64(c_slices)
- gzip_mode = "wb%d" % (options.compression,)
+ gzip_mode = f"wb{options.compression}"
if in_fns:
assert len(out_fns) == c_slices + vars.save_bad
res = c(*cstuff.bytesargs(in_fns, in_msgnames, len(in_fns), out_fns, gzip_mode, minmax_fn, default_value, default_len, default_value_is_None, empty_types_as_None, fmt, fmt_b, record_bad, skip_bad, vars.badmap_fd, vars.badmap_size, vars.save_bad, c_slices, vars.slicemap_fd, vars.slicemap_size, bad_count, default_count, offsets, max_counts))
@@ -529,7 +520,7 @@ def one_column(vars, colname, coltype, out_fns, for_hasher=False):
else:
# python func
if for_hasher:
- raise Exception("Can't hash %s on column of type %s." % (vars.source_name, coltype,))
+ raise Exception(f"Can't hash {vars.source_name} on column of type {coltype}.")
nodefault = object()
if colname in options.defaults:
default_value = options.defaults[colname]
@@ -540,8 +531,6 @@ def one_column(vars, colname, coltype, out_fns, for_hasher=False):
if options.filter_bad:
badmap = mmap(vars.badmap_fd, vars.badmap_size)
vars.map_fhs.append(badmap)
- if PY2:
- badmap = IntegerBytesWrapper(badmap)
if vars.rehashing:
slicemap = mmap(vars.slicemap_fd, vars.slicemap_size)
vars.map_fhs.append(slicemap)
@@ -587,7 +576,7 @@ def one_column(vars, colname, coltype, out_fns, for_hasher=False):
badmap[ix // 8] = bv | (1 << (ix % 8))
continue
else:
- raise Exception("Invalid value %r with no default in %r in %s" % (v, colname, vars.source_name,))
+ raise Exception(f"Invalid value {v!r} with no default in {colname!r} in {vars.source_name}")
if do_minmax and v is not None:
if col_min is None:
col_min = col_max = v
@@ -622,8 +611,8 @@ def print(msg=''):
ds_name = dw.quoted_ds_name
else:
ds_name = quote(dws[0].ds_name[:-1] + '')
- header = '%s -> %s' % (source_name, ds_name)
- builtins.print('%s\n%s' % (header, '=' * len(header)))
+ header = f'{source_name} -> {ds_name}'
+ builtins.print(f"{header}\n{'=' * len(header)}")
header_printed[0] = True
builtins.print(msg)
lines = source.lines
@@ -637,7 +626,7 @@ def print(msg=''):
for colname in columns:
cnt = sum(sum(data[0].get(colname, ())) for data in analysis_res)
if cnt:
- print('%14d %s' % (cnt, colname,))
+ print(f'{cnt:14} {colname}')
for s, cnt in enumerate(bad_line_count_per_slice):
dw_bad.set_lines(s, cnt)
dw_bad.set_compressions('gzip')
@@ -647,15 +636,15 @@ def print(msg=''):
for colname in sorted(options.defaults):
defaulted = [data[2].get(colname, 0) for data in analysis_res]
if sum(defaulted):
- print(' %s:' % (colname,))
+ print(f' {colname}:')
print(' Slice Defaulted line count')
slicecnt = 0
for sliceno, cnt in enumerate(defaulted):
if cnt:
- print(' %5d %d' % (sliceno, cnt,))
+ print(f' {sliceno:5} {cnt}')
slicecnt += 1
if slicecnt > 1:
- print(' total %d' % (sum(defaulted),))
+ print(f' total {sum(defaulted)}')
if dws: # rehashing
if dw: # not as a chain
final_bad_count = [data[1] for data in analysis_res]
diff --git a/accelerator/standard_methods/a_dataset_unroundrobin.py b/accelerator/standard_methods/a_dataset_unroundrobin.py
index 2d5d5018..2b569e6c 100644
--- a/accelerator/standard_methods/a_dataset_unroundrobin.py
+++ b/accelerator/standard_methods/a_dataset_unroundrobin.py
@@ -17,9 +17,6 @@
# #
############################################################################
-from __future__ import division
-from __future__ import absolute_import
-
description = r'''
new_ds.iterate(None) gives the same order as old_ds.iterate('roundrobin').
@@ -50,7 +47,7 @@ def prepare(job):
copy_mode=True,
)
if options.trigger_column:
- assert options.trigger_column in datasets.source.columns, "Trigger column %r not in %s" % (options.trigger_column, datasets.source,)
+ assert options.trigger_column in datasets.source.columns, f"Trigger column {options.trigger_column!r} not in {datasets.source}"
ix = sorted(datasets.source.columns).index(options.trigger_column)
else:
ix = -1
diff --git a/accelerator/standard_methods/c_backend_support.py b/accelerator/standard_methods/c_backend_support.py
index 2f8e29a4..b7df0f94 100644
--- a/accelerator/standard_methods/c_backend_support.py
+++ b/accelerator/standard_methods/c_backend_support.py
@@ -17,16 +17,12 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
import sys
import hashlib
from importlib import import_module
from collections import namedtuple
-from accelerator.compat import unicode, str_types
+from accelerator.compat import str_types
_prologue_code_template = r'''
@@ -115,7 +111,7 @@ def make_source(name, functions, protos, extra_functions, extra_method_defs, wra
method_defs.extend(extra_method_defs)
code.append(_init_code_template % dict(methods=',\n\t'.join(method_defs), name=name,))
hash = hashlib.sha1(b''.join(c.encode('ascii') for c in code)).hexdigest()
- code.insert(-1, 'static char source_hash[] = "%s";\n' % (hash,))
+ code.insert(-1, f'static char source_hash[] = "{hash}";\n')
code = ''.join(code)
return code, hash
@@ -127,11 +123,11 @@ def init(name, hash, protos, extra_protos, functions):
def mk_uint64(count=1):
return [0] * count
def str2c(s):
- if isinstance(s, unicode):
+ if isinstance(s, str):
s = s.encode('utf-8')
return s
else:
- print('[%s] Backend modified since module was compiled, falling back to cffi for development.' % (name,), file=sys.stderr)
+ print(f'[{name}] Backend modified since module was compiled, falling back to cffi for development.', file=sys.stderr)
import cffi
ffi = cffi.FFI()
@@ -139,7 +135,7 @@ def str2c(s):
def mk_uint64(count=1):
return ffi.new('uint64_t []', [0] * count)
def str2c(s):
- if isinstance(s, unicode):
+ if isinstance(s, str):
s = s.encode('utf-8')
return ffi.new('char []', s)
diff --git a/accelerator/standard_methods/csvimport.py b/accelerator/standard_methods/csvimport.py
index d752ac42..09827c58 100644
--- a/accelerator/standard_methods/csvimport.py
+++ b/accelerator/standard_methods/csvimport.py
@@ -20,10 +20,6 @@
# This is a separate file from a_csvimport so setup.py can import
# it and make the _csvimport module at install time.
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
from . import c_backend_support
all_c_functions = r'''
diff --git a/accelerator/standard_methods/dataset_type.py b/accelerator/standard_methods/dataset_type.py
index 078a7eb5..3670780b 100644
--- a/accelerator/standard_methods/dataset_type.py
+++ b/accelerator/standard_methods/dataset_type.py
@@ -21,10 +21,6 @@
# This is a separate file from a_dataset_type so setup.py can import
# it and make the _dataset_type module at install time.
-from __future__ import print_function
-from __future__ import division
-from __future__ import absolute_import
-
from collections import namedtuple
from functools import partial
import sys
@@ -654,8 +650,8 @@ def _test():
known = set(v for v in _convfuncs if ':' not in v)
copy_missing = known - set(copy_types)
copy_extra = set(copy_types) - known
- assert not copy_missing, 'copy_types missing %r' % (copy_missing,)
- assert not copy_extra, 'copy_types contains unexpected %r' % (copy_extra,)
+ assert not copy_missing, f'copy_types missing {copy_missing!r}'
+ assert not copy_extra, f'copy_types contains unexpected {copy_extra!r}'
convert_template = r'''
@@ -1453,7 +1449,7 @@ def _test():
protos.append(proto + ';')
funcs.append(code)
-copy_types = {typerename.get(k.split(':')[0], k.split(':')[0]): 'null_%d' % (v.size,) if v.size else 'null_blob' for k, v in convfuncs.items()}
+copy_types = {typerename.get(k.split(':')[0], k.split(':')[0]): f'null_{v.size}' if v.size else 'null_blob' for k, v in convfuncs.items()}
copy_types['number'] = 'null_number'
copy_types['pickle'] = 'null_blob'
diff --git a/accelerator/statmsg.py b/accelerator/statmsg.py
index f832b6ea..1332ebb1 100644
--- a/accelerator/statmsg.py
+++ b/accelerator/statmsg.py
@@ -34,9 +34,6 @@
# Several built in functions will call this for you, notably dataset
# iterators and pickle load/save functions.
-from __future__ import print_function
-from __future__ import division
-
from contextlib import contextmanager
from errno import ENOTCONN
from functools import partial
@@ -129,16 +126,16 @@ def _start(msg, parent_pid, is_analysis=False):
analysis_cookie = str(_cookie)
else:
analysis_cookie = ''
- _send('start', '%d\0%s\0%s\0%f' % (parent_pid, analysis_cookie, msg, monotonic(),))
+ _send('start', f'{parent_pid}\x00{analysis_cookie}\x00{msg}\x00{monotonic():f}')
def update(msg):
- _send('update', '%s\0\0%s' % (msg, analysis_cookie,))
+ _send('update', f'{msg}\x00\x00{analysis_cookie}')
return update
def _end(pid=None):
_send('end', '', pid=pid)
def _output(pid, msg):
- _send('output', '%f\0%s' % (monotonic(), msg,), pid=pid)
+ _send('output', f'{monotonic():f}\x00{msg}', pid=pid)
def _clear_output(pid):
_send('output', '', pid=pid)
@@ -165,7 +162,7 @@ def fmt(tree, start_indent=0):
current = last[0].summary
if len(last[0].stack) > 1 and not current[1].endswith('analysis'):
msg, t, _ = last[0].stack[1]
- current = (current[0], '%s %s' % (current[1], msg,), t,)
+ current = (current[0], f'{current[1]} {msg}', t,)
except Exception:
print_exc(file=sys.stderr)
res.append((0, 0, 'ERROR', monotonic()))
@@ -177,12 +174,12 @@ def print_status_stacks(stacks=None):
report_t = monotonic()
for pid, indent, msg, t in stacks:
if indent < 0:
- print("%6d TAIL OF OUTPUT: (%.1f seconds ago)" % (pid, report_t - t,))
+ print(f"{pid:6} TAIL OF OUTPUT: ({report_t - t:.1f} seconds ago)")
msgs = list(filter(None, msg.split('\n')))[-3:]
for msg in msgs:
print(" " + colour.green(msg))
else:
- print("%6d STATUS: %s%s (%.1f seconds)" % (pid, " " * indent, msg, report_t - t))
+ print(f"{pid:6} STATUS: {' ' * indent}{msg} ({report_t - t:.1f} seconds)")
def _find(pid, cookie):
@@ -212,7 +209,7 @@ def statmsg_sink(sock):
if ix == len(stack) - 1:
stack.pop()
else:
- print('POP OF WRONG STATUS: %d:%s (index %s of %d)' % (pid, msg, ix, len(stack)))
+ print(f'POP OF WRONG STATUS: {pid}:{msg} (index {ix} of {len(stack)})')
wrong_pops += 1
if wrong_pops == 3:
print('Getting a lot of these? Are you interleaving dataset iterators? Set status_reporting=False on all but one.')
@@ -221,7 +218,7 @@ def statmsg_sink(sock):
msg, _, cookie = msg.split('\0', 3)
stack, ix = _find(pid, cookie)
if ix is None:
- print('UPDATE TO UNKNOWN STATUS %d:%s: %s' % (pid, cookie, msg))
+ print(f'UPDATE TO UNKNOWN STATUS {pid}:{cookie}: {msg}')
else:
stack[ix] = (msg, stack[ix][1], cookie)
elif typ == 'output':
@@ -258,9 +255,9 @@ def statmsg_sink(sock):
del d
status_tree.pop(pid, None)
else:
- print('UNKNOWN MESSAGE: %r' % (data,))
+ print(f'UNKNOWN MESSAGE: {data!r}')
except Exception:
- print('Failed to process %r:' % (data,), file=sys.stderr)
+ print(f'Failed to process {data!r}:', file=sys.stderr)
print_exc(file=sys.stderr)
@@ -281,7 +278,7 @@ def _send(typ, message, pid=None):
if not _send_sock:
fd = int(os.getenv('BD_STATUS_FD'))
_send_sock = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_DGRAM)
- header = ('%s\0%d\0' % (typ, pid or os.getpid(),)).encode('utf-8')
+ header = f'{typ}\x00{pid or os.getpid()}\x00'.encode('utf-8')
message = message.encode('utf-8')
if len(message) > 1450:
message = message[:300] + b'\n....\n' + message[-1100:]
@@ -296,5 +293,5 @@ def _send(typ, message, pid=None):
if e.errno == ENOTCONN:
# The server is dead, no use retrying.
return
- print('Failed to send statmsg (type %s, try %d): %s' % (typ, ix, e))
+ print(f'Failed to send statmsg (type {typ}, try {ix}): {e}')
sleep(0.1 + ix)
diff --git a/accelerator/subjobs.py b/accelerator/subjobs.py
index 5039815e..fc3dbad0 100644
--- a/accelerator/subjobs.py
+++ b/accelerator/subjobs.py
@@ -44,14 +44,14 @@ def build(method, options={}, datasets={}, jobs={}, name=None, caption=None, **k
_bad_kws = set(getarglist(_a.call_method))
bad_kws = _bad_kws & set(kw)
if bad_kws:
- raise Exception('subjobs.build does not accept these keywords: %r' % (bad_kws,))
+ raise Exception(f'subjobs.build does not accept these keywords: {bad_kws!r}')
def run():
return _a.call_method(method, options=options, datasets=datasets, jobs=jobs, record_as=name, caption=caption, **kw)
try:
if name or caption:
- msg = 'Building subjob %s' % (name or method,)
+ msg = f'Building subjob {name or method}'
if caption:
- msg += ' "%s"' % (caption,)
+ msg += f' "{caption}"'
with status(msg):
jid = run()
else:
diff --git a/accelerator/test_methods/a_test_board_metadata.py b/accelerator/test_methods/a_test_board_metadata.py
index 5ce42908..ce1e664c 100644
--- a/accelerator/test_methods/a_test_board_metadata.py
+++ b/accelerator/test_methods/a_test_board_metadata.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test board adding metadata to files and decoding this with "ax sherlock".
'''
@@ -29,7 +25,7 @@
command_prefix=['ax', '--config', '/some/path/here'],
)
-from accelerator.compat import url_quote_more, urlopen, Request, HTTPError, unicode
+from accelerator.compat import url_quote_more, urlopen, Request, HTTPError
from subprocess import Popen, check_output
import os
@@ -77,9 +73,9 @@ def get(filename):
extra_metadata = br'{"job":"/\n/DoES/NoT.EXIST/NOPE","setup_hash":"nah","host":"cough","method":"none","time":0}'
def mk_meta(prefix_a, prefix_b=b'', suffix=b'', offset=0):
data = extra_metadata + suffix
- if isinstance(prefix_a, unicode):
+ if isinstance(prefix_a, str):
return struct.pack(prefix_a, len(data) + len(prefix_b) + offset) + prefix_b + data
- elif isinstance(prefix_b, unicode):
+ elif isinstance(prefix_b, str):
return prefix_a + struct.pack(prefix_b, len(data) + len(prefix_a) + offset) + data
else:
return prefix_a + prefix_b + extra_metadata + suffix
@@ -123,14 +119,14 @@ def crc(data):
fh.write(contents)
modified_contents = get(filename)
if contents == modified_contents:
- raise Exception('%s was not modified by board' % (filename,))
+ raise Exception(f'{filename} was not modified by board')
filename = 'modified.' + filename
with open(filename, 'wb') as fh:
fh.write(modified_contents)
if not any(modified_contents.startswith(v) for v in (want_head if isinstance(want_head, set) else (want_head,))):
- raise Exception('Expected %s to start with %r, but it did not' % (filename, want_head,))
+ raise Exception(f'Expected {filename} to start with {want_head!r}, but it did not')
if not any(modified_contents.endswith(v) for v in (want_tail if isinstance(want_tail, set) else (want_tail,))):
- raise Exception('Expected %s to end with %r, but it did not' % (filename, want_tail,))
+ raise Exception(f'Expected {filename} to end with {want_tail!r}, but it did not')
got = check_output(options.command_prefix + ['sherlock', filename])
got = got.decode('utf-8').strip()
if has_extra_block:
@@ -138,7 +134,7 @@ def crc(data):
else:
want_jobs = job
if got != want_jobs:
- raise Exception('Expected "ax sherlock %s" to give %r, got %r' % (filename, want_jobs, got,))
+ raise Exception(f'Expected "ax sherlock {filename}" to give {want_jobs!r}, got {got!r}')
p.terminate()
p.wait()
diff --git a/accelerator/test_methods/a_test_build_kws.py b/accelerator/test_methods/a_test_build_kws.py
index 85618189..0bd18578 100644
--- a/accelerator/test_methods/a_test_build_kws.py
+++ b/accelerator/test_methods/a_test_build_kws.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
This method is called to test that urd.build finds the correct target for
keywords, and also that it does not accept ambiguous keywords.
diff --git a/accelerator/test_methods/a_test_compare_datasets.py b/accelerator/test_methods/a_test_compare_datasets.py
index 3bbee880..6296c791 100644
--- a/accelerator/test_methods/a_test_compare_datasets.py
+++ b/accelerator/test_methods/a_test_compare_datasets.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
datasets = ("a", "b",)
def analysis(sliceno):
@@ -34,11 +30,11 @@ def analysis(sliceno):
except StopIteration:
try:
next(iter_b)
- raise Exception("dataset b is longer than a in slice %d" % (sliceno,))
+ raise Exception(f"dataset b is longer than a in slice {sliceno}")
except StopIteration:
break
try:
b = next(iter_b)
except StopIteration:
- raise Exception("dataset a is longer than b in slice %d" % (sliceno,))
+ raise Exception(f"dataset a is longer than b in slice {sliceno}")
assert a == b
diff --git a/accelerator/test_methods/a_test_csvexport_all_coltypes.py b/accelerator/test_methods/a_test_csvexport_all_coltypes.py
index 9cb53081..b1a90a95 100644
--- a/accelerator/test_methods/a_test_csvexport_all_coltypes.py
+++ b/accelerator/test_methods/a_test_csvexport_all_coltypes.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify that all column types come out correctly in csvexport.
'''
@@ -29,7 +25,6 @@
from accelerator import subjobs, status
from accelerator.dsutil import _convfuncs
-from accelerator.compat import PY2
def synthesis(job):
dw = job.datasetwriter()
@@ -52,13 +47,9 @@ def synthesis(job):
'unicode',
}
check = {n for n in _convfuncs if not n.startswith('parsed:')}
- assert todo == check, 'Missing/extra column types: %r %r' % (check - todo, todo - check,)
+ assert todo == check, f'Missing/extra column types: {check - todo!r} {todo - check!r}'
for name in sorted(todo):
- if PY2 and name == 'pickle':
- # pickle columns are not supported on python 2.
- t = 'ascii'
- else:
- t = name
+ t = name
dw.add(name, t, none_support=True)
write = dw.get_split_write()
write(
@@ -67,7 +58,7 @@ def synthesis(job):
date(2020, 6, 23), datetime(2020, 6, 23, 12, 13, 14),
1.0, float('-inf'), -10, -20,
{'json': True}, 0xfedcba9876543210beef,
- '...' if PY2 else 1+2j, time(12, 13, 14), 'bl\xe5',
+ 1+2j, time(12, 13, 14), 'bl\xe5',
)
d = {}
d['recursion'] = d
@@ -77,7 +68,7 @@ def synthesis(job):
date(1868, 1, 3), datetime(1868, 1, 3, 13, 14, 5),
float('inf'), float('nan'), 0, 0,
[False, None], 42.18,
- '...' if PY2 else d, time(13, 14, 5), 'bl\xe4',
+ d, time(13, 14, 5), 'bl\xe4',
)
write(
None, None, None,
@@ -131,13 +122,13 @@ def synthesis(job):
'None', 'never', 'None',
)),
):
- with status("Checking with sep=%r, q=%r, none_as=%r" % (sep, q, none_as,)):
+ with status(f"Checking with sep={sep!r}, q={q!r}, none_as={none_as!r}"):
exp = subjobs.build('csvexport', filename='test.csv', separator=sep, source=ds, quote_fields=q, none_as=none_as, lazy_quotes=False)
with exp.open('test.csv', 'r', encoding='utf-8') as fh:
def expect(*a):
want = sep.join(q + v.replace(q, q + q) + q for v in a) + '\n'
got = next(fh)
- assert want == got, 'wanted %r, got %r from %s (export of %s)' % (want, got, exp, ds,)
+ assert want == got, f'wanted {want!r}, got {got!r} from {exp} (export of {ds})'
expect(*sorted(todo))
expect(
'a', 'True', 'hello',
@@ -145,7 +136,7 @@ def expect(*a):
'2020-06-23', '2020-06-23 12:13:14',
'1.0', '-inf', '-10', '-20',
'{"json": true}', '1203552815971897489538799',
- '...' if PY2 else '(1+2j)', '12:13:14', 'bl\xe5',
+ '(1+2j)', '12:13:14', 'bl\xe5',
)
expect(
'b', 'False', 'bye',
@@ -153,6 +144,6 @@ def expect(*a):
'1868-01-03', '1868-01-03 13:14:05',
'inf', 'nan', '0', '0',
'[false, null]', '42.18',
- '...' if PY2 else "{'recursion': {...}}", '13:14:05', 'bl\xe4',
+ "{'recursion': {...}}", '13:14:05', 'bl\xe4',
)
expect(*last_line)
diff --git a/accelerator/test_methods/a_test_csvexport_chains.py b/accelerator/test_methods/a_test_csvexport_chains.py
index 15e64c7a..981062df 100644
--- a/accelerator/test_methods/a_test_csvexport_chains.py
+++ b/accelerator/test_methods/a_test_csvexport_chains.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify that lists of chains are handled in csvexport, including changing
column types.
@@ -32,7 +28,7 @@ def verify(want, **kw):
job = subjobs.build('csvexport', labelsonfirstline=False, **kw)
with job.open('result.csv') as fh:
got = fh.read()
- assert want == got, 'wanted %r, got %r' % (want, got,)
+ assert want == got, f'wanted {want!r}, got {got!r}'
return job
def synthesis(job):
diff --git a/accelerator/test_methods/a_test_csvexport_naming.py b/accelerator/test_methods/a_test_csvexport_naming.py
index 2cbf2ef2..238f54c3 100644
--- a/accelerator/test_methods/a_test_csvexport_naming.py
+++ b/accelerator/test_methods/a_test_csvexport_naming.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify filename (sliced and unsliced) and gzip in csvexport.
'''
@@ -55,4 +51,4 @@ def synthesis(job):
want = b'a\n1\n'
else:
want = b'a\n0\n1\n2\n'
- assert want == got, 'wanted %r, got %r in %s' % (want, got, fn)
+ assert want == got, f'wanted {want!r}, got {got!r} in {fn}'
diff --git a/accelerator/test_methods/a_test_csvexport_quoting.py b/accelerator/test_methods/a_test_csvexport_quoting.py
index 967903be..22ba2e3a 100644
--- a/accelerator/test_methods/a_test_csvexport_quoting.py
+++ b/accelerator/test_methods/a_test_csvexport_quoting.py
@@ -23,15 +23,13 @@
'''
from accelerator import subjobs
-from accelerator.compat import PY3
def test(job, prefix, none_support):
expect = [[], [], []]
def write(sliceno, a, b, c):
w(a, b, c)
if isinstance(c, bytes):
- if PY3:
- c = c.decode('utf-8', 'backslashreplace')
+ c = c.decode('utf-8', 'backslashreplace')
else:
c = repr(c)
expect[sliceno].append((str(a), repr(b), c,))
@@ -76,7 +74,7 @@ def verify(source, lazy_quotes, q, sep, expect, **kw):
separator=sep,
**kw
)
- with j.open('result.csv', 'r' if PY3 else 'rb') as fh:
+ with j.open('result.csv', 'r') as fh:
got = fh.read()
if lazy_quotes and sep:
quote_func = make_lazy(sep, q)
@@ -84,13 +82,13 @@ def verify(source, lazy_quotes, q, sep, expect, **kw):
quote_func = lambda v: q + v.replace(q, q + q) + q
want = '\n'.join(sep.join(map(quote_func, line)) for line in expect)
if want != got:
- print('Unhappy with %s:' % (j.filename('result.csv'),))
+ print(f"Unhappy with {j.filename('result.csv')}:")
print()
print('Expected:')
print(want)
print('Got:')
print(got)
- raise Exception('csvexport failed with quote_fields=%r, separator=%r, lazy_quotes=%r' % (q, sep, lazy_quotes,))
+ raise Exception(f'csvexport failed with quote_fields={q!r}, separator={sep!r}, lazy_quotes={lazy_quotes!r}')
def make_lazy(sep, q):
if q == '"':
diff --git a/accelerator/test_methods/a_test_csvexport_separators.py b/accelerator/test_methods/a_test_csvexport_separators.py
index a3afda67..271582b9 100644
--- a/accelerator/test_methods/a_test_csvexport_separators.py
+++ b/accelerator/test_methods/a_test_csvexport_separators.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test some strange choices for separators in csvexport.
'''
@@ -49,7 +45,7 @@ def verify(data, filename):
open_func = open
with open_func(j.filename(filename), 'rb') as fh:
got = fh.read()
- assert want == got, "Expected %s/%s to contain %r, but contained %r" % (j, filename, want, got,)
+ assert want == got, f"Expected {j}/{filename} to contain {want!r}, but contained {got!r}"
for separator in ('', '\0', 'wheeee'):
for line_separator in ('', '\0', 'woooooo'):
for quote in ('', 'qqq'):
diff --git a/accelerator/test_methods/a_test_csvimport_corner_cases.py b/accelerator/test_methods/a_test_csvimport_corner_cases.py
index f949055f..7573b717 100644
--- a/accelerator/test_methods/a_test_csvimport_corner_cases.py
+++ b/accelerator/test_methods/a_test_csvimport_corner_cases.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify various corner cases in csvimport.
'''
@@ -31,10 +27,10 @@
from accelerator import subjobs
from accelerator.dispatch import JobError
from accelerator.dataset import Dataset
-from accelerator.compat import PY3, uni
+from accelerator.compat import uni
def openx(filename):
- return open(filename, "xb" if PY3 else "wbx")
+ return open(filename, "xb")
def check_array(job, lines, filename, bad_lines=(), **options):
d = {}
@@ -74,31 +70,31 @@ def verify_ds(options, d, d_bad, d_skipped, filename, d_columns=None):
except ValueError:
# We have a few non-numeric ones
pass
- assert ix in d, "Bad index %r in %r (%s)" % (ix, filename, jid,)
- assert a == b == d[ix], "Wrong data for line %r in %r (%s)" % (ix, filename, jid,)
+ assert ix in d, f"Bad index {ix!r} in {filename!r} ({jid})"
+ assert a == b == d[ix], f"Wrong data for line {ix!r} in {filename!r} ({jid})"
del d[ix]
- assert not d, "Not all lines returned from %r (%s), %r missing" % (filename, jid, set(d.keys()),)
+ assert not d, f"Not all lines returned from {filename!r} ({jid}), {set(d.keys())!r} missing"
if options.get("allow_bad"):
bad_ds = Dataset(jid, "bad")
for ix, data in bad_ds.iterate(None, ["lineno", "data"]):
- assert ix in d_bad, "Bad bad_lineno %d in %r (%s/bad) %r" % (ix, filename, jid, data,)
- assert data == d_bad[ix], "Wrong saved bad line %d in %r (%s/bad).\nWanted %r.\nGot %r." % (ix, filename, jid, d_bad[ix], data,)
+ assert ix in d_bad, f"Bad bad_lineno {ix} in {filename!r} ({jid}/bad) {data!r}"
+ assert data == d_bad[ix], f"Wrong saved bad line {ix} in {filename!r} ({jid}/bad).\nWanted {d_bad[ix]!r}.\nGot {data!r}."
del d_bad[ix]
verify_minmax(bad_ds, "lineno")
- assert not d_bad, "Not all bad lines returned from %r (%s), %r missing" % (filename, jid, set(d_bad.keys()),)
+ assert not d_bad, f"Not all bad lines returned from {filename!r} ({jid}), {set(d_bad.keys())!r} missing"
if options.get("comment") or options.get("skip_lines"):
skipped_ds = Dataset(jid, "skipped")
for ix, data in skipped_ds.iterate(None, ["lineno", "data"]):
- assert ix in d_skipped, "Bad skipped_lineno %d in %r (%s/skipped) %r" % (ix, filename, jid, data,)
- assert data == d_skipped[ix], "Wrong saved skipped line %d in %r (%s/skipped).\nWanted %r.\nGot %r." % (ix, filename, jid, d_skipped[ix], data,)
+ assert ix in d_skipped, f"Bad skipped_lineno {ix} in {filename!r} ({jid}/skipped) {data!r}"
+ assert data == d_skipped[ix], f"Wrong saved skipped line {ix} in {filename!r} ({jid}/skipped).\nWanted {d_skipped[ix]!r}.\nGot {data!r}."
del d_skipped[ix]
verify_minmax(skipped_ds, "lineno")
- assert not d_skipped, "Not all bad lines returned from %r (%s), %r missing" % (filename, jid, set(d_skipped.keys()),)
+ assert not d_skipped, f"Not all bad lines returned from {filename!r} ({jid}), {set(d_skipped.keys())!r} missing"
if options.get("lineno_label"):
lineno_got = dict(ds.iterate(None, [d_columns[0], options.get("lineno_label")]))
- assert lineno_got == lineno_want, "%r != %r" % (lineno_got, lineno_want,)
+ assert lineno_got == lineno_want, f"{lineno_got!r} != {lineno_want!r}"
verify_minmax(ds, options["lineno_label"])
def verify_minmax(ds, colname):
@@ -106,14 +102,14 @@ def verify_minmax(ds, colname):
minmax_want = (min(data) , max(data),) if data else (None, None,)
col = ds.columns[colname]
minmax_got = (col.min, col.max,)
- assert minmax_got == minmax_want, "%s: %r != %r" % (ds, minmax_got, minmax_want,)
+ assert minmax_got == minmax_want, f"{ds}: {minmax_got!r} != {minmax_want!r}"
def require_failure(name, options):
try:
subjobs.build("csvimport", options=options)
except JobError:
return
- raise Exception("File with %s was imported without error." % (name,))
+ raise Exception(f"File with {name} was imported without error.")
def check_bad_file(job, name, data):
filename = name + ".txt"
@@ -124,11 +120,8 @@ def check_bad_file(job, name, data):
)
require_failure(name, options)
-if PY3:
- def bytechr(i):
- return chr(i).encode("iso-8859-1")
-else:
- bytechr = chr
+def bytechr(i):
+ return chr(i).encode("iso-8859-1")
def byteline(start, stop, nl, q):
s = b''.join(bytechr(i) for i in range(start, stop) if i != nl)
@@ -150,7 +143,7 @@ def write(data):
for q in (None, 0, 34, 13, 10, 228):
if nl == q:
continue
- filename = "no separator.%r.%r.txt" % (nl, q,)
+ filename = f"no separator.{nl!r}.{q!r}.txt"
nl_b = bytechr(nl)
q_b = bytechr(q) if q else b''
wrote_c = Counter()
@@ -167,9 +160,9 @@ def write(data):
labels=["data"],
))
except JobError:
- raise Exception("Importing %r failed" % (filename,))
+ raise Exception(f"Importing {filename!r} failed")
got_c = Counter(Dataset(jid).iterate(None, "data"))
- assert got_c == wrote_c, "Importing %r (%s) gave wrong contents" % (filename, jid,)
+ assert got_c == wrote_c, f"Importing {filename!r} ({jid}) gave wrong contents"
def check_good_file(job, name, data, d, d_bad={}, d_skipped={}, d_columns=None, **options):
filename = name + ".txt"
diff --git a/accelerator/test_methods/a_test_csvimport_separators.py b/accelerator/test_methods/a_test_csvimport_separators.py
index f2f78d08..3ba20ccb 100644
--- a/accelerator/test_methods/a_test_csvimport_separators.py
+++ b/accelerator/test_methods/a_test_csvimport_separators.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify that various separators and line endings work in csvimport,
with and without quoting.
@@ -72,14 +68,14 @@ def check_one(job, newline, sep, data, want_res=None, prefix="", quotes=False, l
newline='' if "\n" in newline else newline,
))
except JobError as e:
- raise CSVImportException("Failed to csvimport for separator %d with newline %r, csvimport error was:\n%s" % (sep, newline, e.format_msg()))
+ raise CSVImportException(f"Failed to csvimport for separator {sep} with newline {newline!r}, csvimport error was:\n{e.format_msg()}")
ds = Dataset(jid)
labels = sorted(ds.columns)
if labels != data[0]:
- raise WrongLabelsException("csvimport gave wrong labels for separator %d with newline %r: %r (expected %r)" % (sep, newline, labels, data[0],))
+ raise WrongLabelsException(f"csvimport gave wrong labels for separator {sep} with newline {newline!r}: {labels!r} (expected {data[0]!r})")
res = list(ds.iterate(None, data[0]))
if res != want_res:
- raise WrongDataException("csvimport gave wrong data for separator %d with newline %r: %r (expected %r)" % (sep, newline, res, want_res,))
+ raise WrongDataException(f"csvimport gave wrong data for separator {sep} with newline {newline!r}: {res!r} (expected {want_res!r})")
def synthesis(job):
# Any iso-8859-1 character is a valid separator, but let's try
diff --git a/accelerator/test_methods/a_test_csvimport_slicing.py b/accelerator/test_methods/a_test_csvimport_slicing.py
index 1e84204a..75dabe92 100644
--- a/accelerator/test_methods/a_test_csvimport_slicing.py
+++ b/accelerator/test_methods/a_test_csvimport_slicing.py
@@ -17,8 +17,6 @@
# #
############################################################################
-from __future__ import unicode_literals
-
description = r'''
Verify that csvimport puts lines in the expected slice.
@@ -53,8 +51,8 @@ def add_comment(line):
if sometimes and randint(0, 9) == 3:
add_comment(sometimes % (ix, sliceno,))
if comments and sliceno % comments == 0:
- add_comment('# line %d, before %d,%d' % (lineno[0], ix, sliceno,))
- fh.write('%d,%d\n' % (ix, sliceno,))
+ add_comment(f'# line {lineno[0]}, before {ix},{sliceno}')
+ fh.write(f'{ix},{sliceno}\n')
want_linenos[sliceno].append(lineno[0])
lineno[0] += 1
job = subjobs.build(
@@ -69,16 +67,16 @@ def add_comment(line):
ds = job.dataset()
for sliceno in range(job.params.slices):
got = [int(x) for x in ds.iterate(sliceno, 'sliceno')]
- assert got == [sliceno] * 10, "Slice %d has wrong slices in %s: %r" % (sliceno, ds.quoted, got,)
+ assert got == [sliceno] * 10, f"Slice {sliceno} has wrong slices in {ds.quoted}: {got!r}"
got = [int(x) for x in ds.iterate(sliceno, 'ix')]
- assert got == list(range(10)), "Slice %d has wrong ixes in %s: %r" % (sliceno, ds.quoted, got,)
+ assert got == list(range(10)), f"Slice {sliceno} has wrong ixes in {ds.quoted}: {got!r}"
got = list(ds.iterate(sliceno, 'lineno'))
- assert got == want_linenos[sliceno], "Slice %d has wrong lines in %s:\n wanted %r\n got %r" % (sliceno, ds.quoted, want_linenos[sliceno], got,)
+ assert got == want_linenos[sliceno], f"Slice {sliceno} has wrong lines in {ds.quoted}:\n wanted {want_linenos[sliceno]!r}\n got {got!r}"
if comments or sometimes or initial_skipped:
ds = job.dataset('skipped')
for sliceno in range(job.params.slices):
got = list(ds.iterate(sliceno, ('lineno', 'data')))
- assert got == want_comments[sliceno], "Slice %d has wrong skipped lines in %s:\n wanted %r\n got %r" % (sliceno, ds.quoted, want_comments[sliceno], got,)
+ assert got == want_comments[sliceno], f"Slice {sliceno} has wrong skipped lines in {ds.quoted}:\n wanted {want_comments[sliceno]!r}\n got {got!r}"
def synthesis(job):
diff --git a/accelerator/test_methods/a_test_csvimport_zip.py b/accelerator/test_methods/a_test_csvimport_zip.py
index 217d3278..9f80a364 100644
--- a/accelerator/test_methods/a_test_csvimport_zip.py
+++ b/accelerator/test_methods/a_test_csvimport_zip.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify the zip wrapper for csvimport.
'''
@@ -55,7 +51,7 @@ def verify(zipname, inside_filenames, want_ds, **kw):
jid = subjobs.build('csvimport_zip', options=opts)
for dsn, want_data in want_ds.items():
got_data = list(Dataset(jid, dsn).iterate(None, '0'))
- assert got_data == want_data, "%s/%s from %s didn't contain %r, instead contained %r" % (jid, dsn, zipname, want_data, got_data)
+ assert got_data == want_data, f"{jid}/{dsn} from {zipname} didn't contain {want_data!r}, instead contained {got_data!r}"
def verify_order(want_order, namemap={}, **kw):
opts=dict(
@@ -69,13 +65,13 @@ def verify_order(want_order, namemap={}, **kw):
for dsn in want_order:
b = b'contents of ' + namemap.get(dsn, dsn).encode('ascii')
got_data = list(Dataset(jid, dsn).iterate(None, '0'))
- assert got_data == [b], "%s/%s from 'many files.zip' didn't contain [%r], instead contained %r" % (jid, dsn, b, got_data)
+ assert got_data == [b], f"{jid}/{dsn} from 'many files.zip' didn't contain [{b!r}], instead contained {got_data!r}"
got_order = [ds.name for ds in Dataset(jid, want_order[-1]).chain()]
- assert want_order == got_order, 'Wanted order %r, got %r in %s' % (want_order, got_order, jid,)
+ assert want_order == got_order, f'Wanted order {want_order!r}, got {got_order!r} in {jid}'
if len(want_order) > 1: # chaining is actually on, so a default ds is produced
got_order = [ds.name for ds in Dataset(jid).chain()]
want_order[-1] = 'default'
- assert want_order == got_order, 'Wanted order %r, got %r in %s' % (want_order, got_order, jid,)
+ assert want_order == got_order, f'Wanted order {want_order!r}, got {got_order!r} in {jid}'
def synthesis():
# Simple case, a single file in the zip.
diff --git a/accelerator/test_methods/a_test_dataset_callbacks.py b/accelerator/test_methods/a_test_dataset_callbacks.py
index 0e2ba22e..eb6821a9 100644
--- a/accelerator/test_methods/a_test_dataset_callbacks.py
+++ b/accelerator/test_methods/a_test_dataset_callbacks.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Tests many variations of dataset iteration callbacks with skipping.
'''
@@ -161,7 +157,7 @@ def chk_both_post(ds):
if slices == 3 and ds.name == '3':
# ('3', 3) never happens, so fake it.
current_expect.append(5)
- assert current_expect == current, '%s %r %r'%(ds,current_expect, current)
+ assert current_expect == current, f'{ds} {current_expect!r} {current!r}'
for v in ds4.iterate_chain(None, 'a', pre_callback=chk_both_pre, post_callback=chk_both_post):
current.append(v)
assert seen_pre == set.union(*(set((n, s) for s in range(slices)) for n in '1234'))
diff --git a/accelerator/test_methods/a_test_dataset_checksum.py b/accelerator/test_methods/a_test_dataset_checksum.py
index 83d5a18b..c0e2d29a 100644
--- a/accelerator/test_methods/a_test_dataset_checksum.py
+++ b/accelerator/test_methods/a_test_dataset_checksum.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test dataset_checksum[_chain].
'''
@@ -28,7 +24,6 @@
from accelerator.dataset import DatasetWriter
from accelerator import subjobs
from accelerator import blob
-from accelerator.compat import PY3
test_data = [
("a", b"A", b"0", 0.42, 18, [1, 2, 3], u"a", u"A"),
@@ -48,12 +43,11 @@ def prepare():
unicode="unicode",
unicode_none=("unicode", True),
)
- if PY3:
- # z so it sorts last
- columns['zpickle'] = 'pickle'
- for ix, v in enumerate(test_data):
- test_data[ix] = v + ([ix, 'line %d' % (ix,), {'line': ix}, 42],)
- test_data[-1][-1][-1] = float('-inf')
+ # z so it sorts last
+ columns['zpickle'] = 'pickle'
+ for ix, v in enumerate(test_data):
+ test_data[ix] = v + ([ix, f'line {ix}', {'line': ix}, 42],)
+ test_data[-1][-1][-1] = float('-inf')
a = DatasetWriter(name="a", columns=columns)
b = DatasetWriter(name="b", columns=columns, previous=a)
c = DatasetWriter(name="c", columns=columns)
@@ -99,11 +93,10 @@ def synthesis(prepare_res):
a_uns_sum = ck(a, sort=False)
b_uns_sum = ck(b, sort=False)
assert a_uns_sum != b_uns_sum # they are not the same order
- if PY3:
- # Check that the pickle column really was included and works.
- a_p_sum = ck(a, columns={'zpickle'})
- b_p_sum = ck(b, columns={'zpickle'})
- assert a_p_sum == b_p_sum # same values
- a_uns_p_sum = ck(a, columns={'zpickle'}, sort=False)
- b_uns_p_sum = ck(b, columns={'zpickle'}, sort=False)
- assert a_uns_p_sum != b_uns_p_sum # but they are not the same order
+ # Check that the pickle column really was included and works.
+ a_p_sum = ck(a, columns={'zpickle'})
+ b_p_sum = ck(b, columns={'zpickle'})
+ assert a_p_sum == b_p_sum # same values
+ a_uns_p_sum = ck(a, columns={'zpickle'}, sort=False)
+ b_uns_p_sum = ck(b, columns={'zpickle'}, sort=False)
+ assert a_uns_p_sum != b_uns_p_sum # but they are not the same order
diff --git a/accelerator/test_methods/a_test_dataset_column_names.py b/accelerator/test_methods/a_test_dataset_column_names.py
index 312ad399..afde6eac 100644
--- a/accelerator/test_methods/a_test_dataset_column_names.py
+++ b/accelerator/test_methods/a_test_dataset_column_names.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test writing datasets with strange column names, column names whose cleaned
names collide and column names used in the generated split_write function.
@@ -40,14 +36,14 @@ def prepare():
return mk_dw("internal_names_analysis", internal_names_analysis)
def analysis(sliceno, prepare_res):
- prepare_res.write(*['a %d' % (sliceno,)] * 5)
- prepare_res.write_list(['b %d' % (sliceno,)] * 5)
- prepare_res.write_dict(dict(zip(internal_names_analysis, ['c %d' % (sliceno,)] * 5)))
+ prepare_res.write(*[f'a {sliceno}'] * 5)
+ prepare_res.write_list([f'b {sliceno}'] * 5)
+ prepare_res.write_dict(dict(zip(internal_names_analysis, [f'c {sliceno}'] * 5)))
def synthesis(prepare_res, slices):
ds = prepare_res.finish()
for sliceno in range(slices):
- assert list(ds.iterate(sliceno, internal_names_analysis)) == [('a %d' % (sliceno,),) * 5, ('b %d' % (sliceno,),) *5, ('c %d' % (sliceno,),) *5]
+ assert list(ds.iterate(sliceno, internal_names_analysis)) == [(f'a {sliceno}',) * 5, (f'b {sliceno}',) *5, (f'c {sliceno}',) *5]
in_parent = [ # list because order matters
"-", # becomes _ because everything must be a valid python identifier.
@@ -76,7 +72,7 @@ def synthesis(prepare_res, slices):
child = dw.finish()
for colname in in_parent + in_child:
data = set(child.iterate(None, colname))
- assert data == {colname + " 1", colname + " 2"}, "Bad data for %s: %r" % (colname, data)
+ assert data == {colname + " 1", colname + " 2"}, f"Bad data for {colname}: {data!r}"
def chk_internal(name, **kw):
internal = ("writers", "w_l", "cyc", "hsh", "next",)
diff --git a/accelerator/test_methods/a_test_dataset_concat.py b/accelerator/test_methods/a_test_dataset_concat.py
index a04f84a0..45732dac 100644
--- a/accelerator/test_methods/a_test_dataset_concat.py
+++ b/accelerator/test_methods/a_test_dataset_concat.py
@@ -25,7 +25,6 @@
from itertools import chain
from accelerator import subjobs, JobError
-from accelerator.compat import PY2
from accelerator.dsutil import _type2iter
def synthesis(job):
@@ -50,12 +49,10 @@ def synthesis(job):
}
missing = set(_type2iter) - set(types.values())
assert not missing, missing
- if PY2:
- del types['n'] # no pickle type on python2
def data(ix):
d = {
- 'a': '%d' % (ix,),
+ 'a': f'{ix}',
'b': bool(ix % 2),
'c': b'%d' % (ix,),
'd': complex(0, ix),
@@ -70,7 +67,7 @@ def data(ix):
'm': -1.0 / (ix + 1),
'n': {ix},
'o': time(0, ix // 60 % 60, ix % 60),
- 'p': u'%d' % (ix,),
+ 'p': f'{ix}',
'extra': 0,
}
return {k: v for k, v in d.items() if k in types}
@@ -126,7 +123,7 @@ def chk(source, previous, want_in_chain, want, do_sort=True, none_support=()):
def want_fail(why, **kw):
try:
subjobs.build('dataset_concat', **kw)
- raise Exception('dataset_concat(%r) should have failed: %s' % (kw, why,))
+ raise Exception(f'dataset_concat({kw!r}) should have failed: {why}')
except JobError:
pass
diff --git a/accelerator/test_methods/a_test_dataset_empty_colname.py b/accelerator/test_methods/a_test_dataset_empty_colname.py
index dcce6a0e..234d7579 100644
--- a/accelerator/test_methods/a_test_dataset_empty_colname.py
+++ b/accelerator/test_methods/a_test_dataset_empty_colname.py
@@ -52,7 +52,7 @@ def chk_order(names, want):
ds = dw.finish()
for col in want:
# '' hashes to 0, so if the hashlabel worked both are in slice 0.
- assert list(ds.iterate(0, col)) == [col, col], '%r bad in %s' % (col, ds,)
+ assert list(ds.iterate(0, col)) == [col, col], f'{col!r} bad in {ds}'
return ds
chk_order(['@', '_', ''], {'@': '_', '_': '__', '': '___'})
chk_order(['@', '', '_'], {'@': '_', '': '__', '_': '___'})
@@ -68,7 +68,7 @@ def chk_mismatch(ds, good, bad):
except DatasetUsageError:
pass
else:
- raise Exception('%s accepted hashlabel %r' % (ds, bad,))
+ raise Exception(f'{ds} accepted hashlabel {bad!r}')
chk_mismatch(ds, '', '@')
dw = job.datasetwriter(name='hl_', hashlabel='_')
dw.add('_', 'int32')
diff --git a/accelerator/test_methods/a_test_dataset_fanout.py b/accelerator/test_methods/a_test_dataset_fanout.py
index 1919eff3..7bc7a319 100644
--- a/accelerator/test_methods/a_test_dataset_fanout.py
+++ b/accelerator/test_methods/a_test_dataset_fanout.py
@@ -17,16 +17,11 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test dataset_fanout with varying types, hashlabel and chain truncation.
'''
from accelerator import subjobs
-from accelerator.compat import unicode
from itertools import cycle
@@ -42,19 +37,19 @@ def mk(name, types, lines, hashlabel=None, previous=None):
def chk(job, colnames, types, ds2lines, previous={}, hashlabel=None):
have_ds = set(ds.name for ds in job.datasets)
want_ds = set(ds2lines)
- assert have_ds == want_ds, 'Job %r should have had datasets %r but had %r' % (job, want_ds, have_ds,)
+ assert have_ds == want_ds, f'Job {job!r} should have had datasets {want_ds!r} but had {have_ds!r}'
colnames = sorted(colnames)
for ds, lines in ds2lines.items():
ds = job.dataset(ds)
- assert ds.hashlabel == hashlabel, 'Dataset %s should have had hashlabel %s but had %s' % (ds.quoted, hashlabel, ds.hashlabel,)
- assert ds.previous == previous.get(ds.name), 'Dataset %s should have had previous %s but had %s' % (ds.quoted, previous.get(ds.name), ds.previous,)
+ assert ds.hashlabel == hashlabel, f'Dataset {ds.quoted} should have had hashlabel {hashlabel} but had {ds.hashlabel}'
+ assert ds.previous == previous.get(ds.name), f'Dataset {ds.quoted} should have had previous {previous.get(ds.name)} but had {ds.previous}'
ds_colnames = sorted(ds.columns)
- assert ds_colnames == colnames, 'Dataset %s should have had columns %r but had %r' % (ds.quoted, colnames, ds_colnames,)
+ assert ds_colnames == colnames, f'Dataset {ds.quoted} should have had columns {colnames!r} but had {ds_colnames!r}'
ds_types = tuple(col.type for _, col in sorted(ds.columns.items()))
- assert ds_types == types, 'Dataset %s should have had columns with types %r but had %r' % (ds.quoted, types, ds_types,)
+ assert ds_types == types, f'Dataset {ds.quoted} should have had columns with types {types!r} but had {ds_types!r}'
have_lines = sorted(ds.iterate(None))
want_lines = sorted(lines)
- assert have_lines == want_lines, 'Dataset %s should have contained %r but contained %r' % (ds.quoted, want_lines, have_lines,)
+ assert have_lines == want_lines, f'Dataset {ds.quoted} should have contained {want_lines!r} but contained {have_lines!r}'
# just a simple splitting
a = mk('a', ('unicode', 'ascii', 'int64'), [('a', 'a', 1), ('b', 'b', 2), ('a', 'c', 3)], hashlabel='A')
@@ -184,10 +179,10 @@ def chk(job, colnames, types, ds2lines, previous={}, hashlabel=None):
cycle(['int64', 'int32']),
cycle(['unicode', 'ascii']),
)):
- data = [('data',) + (ix + 1000,) * 4 + (unicode(ix),)]
+ data = [('data',) + (ix + 1000,) * 4 + (str(ix),)]
want_data.append(data[0][1:])
all_types.append(
- mk('all types %d' % (ix,), types, data, previous=previous)
+ mk(f'all types {ix}', types, data, previous=previous)
)
previous = all_types[-1]
diff --git a/accelerator/test_methods/a_test_dataset_filter_columns.py b/accelerator/test_methods/a_test_dataset_filter_columns.py
index 63654f0c..1a1bba17 100644
--- a/accelerator/test_methods/a_test_dataset_filter_columns.py
+++ b/accelerator/test_methods/a_test_dataset_filter_columns.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test the dataset_filter_columns method.
'''
@@ -41,10 +37,10 @@ def chk(j, *want):
ds = j.dataset()
want = set(want)
got = set(ds.columns)
- assert got == want, "%s should have had columns %r but had %r" % (ds, want, got,)
+ assert got == want, f"{ds} should have had columns {want!r} but had {got!r}"
want = list(zip(*[(ord(c) - 96, ord(c)) for c in sorted(want)]))
got = list(ds.iterate(None))
- assert got == want, "%s should have had %r but had %r" % (ds, want, got,)
+ assert got == want, f"{ds} should have had {want!r} but had {got!r}"
chk(job, 'a', 'b', 'c', 'd')
j = subjobs.build('dataset_filter_columns', source=ds, keep_columns=['a'])
chk(j, 'a')
diff --git a/accelerator/test_methods/a_test_dataset_in_prepare.py b/accelerator/test_methods/a_test_dataset_in_prepare.py
index ebf380de..82c980ca 100644
--- a/accelerator/test_methods/a_test_dataset_in_prepare.py
+++ b/accelerator/test_methods/a_test_dataset_in_prepare.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test writing a dataset in prepare, verifying that it is usable in
analysis and synthesis with no manual .finish()
diff --git a/accelerator/test_methods/a_test_dataset_merge.py b/accelerator/test_methods/a_test_dataset_merge.py
index 7c8e0d5e..6f7e99c6 100644
--- a/accelerator/test_methods/a_test_dataset_merge.py
+++ b/accelerator/test_methods/a_test_dataset_merge.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test Dataset.merge() and the dataset_merge method.
'''
@@ -49,7 +45,7 @@ def fail_merge(a, b, **kw):
a.merge(b, name='failme', **kw)
except DatasetUsageError:
return
- raise Exception("Merging %s and %s with %r didn't fail as it should have" % (a, b, kw,))
+ raise Exception(f"Merging {a} and {b} with {kw!r} didn't fail as it should have")
checks = {}
def check(ds, want):
@@ -57,7 +53,7 @@ def check(ds, want):
checks[ds.name] = want
got = list(ds.iterate_chain(None))
got.sort()
- assert got == want, "%s contained %r not %r as expetected" % (ds, got, want,)
+ assert got == want, f"{ds} contained {got!r} not {want!r} as expetected"
def synthesis(params):
a0 = mkds('a0', ['0', '1'], [(1, 2), (3, 4), (5, 6)])
@@ -147,4 +143,4 @@ def synthesis(params):
subjobs.build('dataset_merge', datasets=dict(source=parents), options=kw)
except JobError:
continue
- raise Exception("dataset_merge incorrectly allowed %r with %r" % (parents, kw))
+ raise Exception(f"dataset_merge incorrectly allowed {parents!r} with {kw!r}")
diff --git a/accelerator/test_methods/a_test_dataset_names.py b/accelerator/test_methods/a_test_dataset_names.py
index 4ad15061..844f60dc 100644
--- a/accelerator/test_methods/a_test_dataset_names.py
+++ b/accelerator/test_methods/a_test_dataset_names.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify that some potentially problematic dataset names work.
'''
diff --git a/accelerator/test_methods/a_test_dataset_nan.py b/accelerator/test_methods/a_test_dataset_nan.py
index 7010b871..10253152 100644
--- a/accelerator/test_methods/a_test_dataset_nan.py
+++ b/accelerator/test_methods/a_test_dataset_nan.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test that NaN does not end up in min/max unless it's the only value.
'''
@@ -63,8 +59,8 @@ def check(dw, want_min, want_max):
ds = dw.finish()
for colname, want_min, want_max in zip(['float32', 'float64', 'number'], want_min, want_max):
col = ds.columns[colname]
- assert eq(col.min, want_min), "%s.%s should have had min value %s, but had %s" % (ds, colname, want_min, col.min)
- assert eq(col.max, want_max), "%s.%s should have had max value %s, but had %s" % (ds, colname, want_max, col.max)
+ assert eq(col.min, want_min), f"{ds}.{colname} should have had min value {want_min}, but had {col.min}"
+ assert eq(col.max, want_max), f"{ds}.{colname} should have had max value {want_max}, but had {col.max}"
check(a, [nan, nan, nan], [nan, nan, nan])
check(b, [nan, 2, nan], [nan, 2, nan])
check(c, [0, 1, 1], [inf, 1, 2])
diff --git a/accelerator/test_methods/a_test_dataset_overwrite.py b/accelerator/test_methods/a_test_dataset_overwrite.py
index 9b666937..8818b946 100644
--- a/accelerator/test_methods/a_test_dataset_overwrite.py
+++ b/accelerator/test_methods/a_test_dataset_overwrite.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify that datasets can not be overwritten.
'''
diff --git a/accelerator/test_methods/a_test_dataset_parsing_writer.py b/accelerator/test_methods/a_test_dataset_parsing_writer.py
index c2645019..4c47be83 100644
--- a/accelerator/test_methods/a_test_dataset_parsing_writer.py
+++ b/accelerator/test_methods/a_test_dataset_parsing_writer.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Tests the parsed:type writers.
With plain values, parsable values and unparsable values.
@@ -53,18 +49,18 @@ def test(typ, write_values, want_values, bad_values=[]):
try:
write(value)
except Exception as e:
- raise Exception('Failed to write %r to %s column: %r' % (value, typ, e))
+ raise Exception(f'Failed to write {value!r} to {typ} column: {e!r}')
for value in ['foo', '1 two'] + bad_values:
try:
write(value)
- raise Exception('writer for parsed:%s allowed %r' % (typ, value,))
+ raise Exception(f'writer for parsed:{typ} allowed {value!r}')
except (ValueError, OverflowError):
pass
ds = dw.finish()
for sliceno in range(slices):
want_slice = hashfilter(typ, want_values, sliceno)
got_slice = list(ds.iterate(sliceno, 'value'))
- assert nanfix(got_slice) == nanfix(want_slice), "%s got %r, wanted %r in slice %d" % (typ, got_slice, want_slice, sliceno)
+ assert nanfix(got_slice) == nanfix(want_slice), f"{typ} got {got_slice!r}, wanted {want_slice!r} in slice {sliceno}"
inf = float('inf')
nan = float('nan')
diff --git a/accelerator/test_methods/a_test_dataset_range.py b/accelerator/test_methods/a_test_dataset_range.py
index 6a9d6d6e..75fb4c53 100644
--- a/accelerator/test_methods/a_test_dataset_range.py
+++ b/accelerator/test_methods/a_test_dataset_range.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test the range argument to the dataset iteration functions
and chain.with_column().
diff --git a/accelerator/test_methods/a_test_dataset_rename_columns.py b/accelerator/test_methods/a_test_dataset_rename_columns.py
index 0ae39233..0e0ffa5b 100644
--- a/accelerator/test_methods/a_test_dataset_rename_columns.py
+++ b/accelerator/test_methods/a_test_dataset_rename_columns.py
@@ -41,7 +41,7 @@ def synthesis(job):
dw = job.datasetwriter(name='b', hashlabel='b', columns=columns, previous=a)
dw.get_split_write()(1, 2, 3)
b = dw.finish()
- names = ('link%d' % (ix,) for ix in range(1000)) # more than enough
+ names = (f'link{ix}' for ix in range(1000)) # more than enough
def chk(ds, want_hashlabel, want_previous, want_coltypes, rename):
got_job = subjobs.build('dataset_rename_columns', rename=rename, source=ds)
chk_inner(got_job.dataset(), want_hashlabel, want_previous, want_coltypes)
@@ -53,9 +53,9 @@ def chk_inner(got_ds, want_hashlabel, want_previous, want_coltypes):
got_cols = set(got_ds.columns)
want_cols = set(want_coltypes)
extra = got_cols - want_cols
- assert not extra, 'got extra columns %r' % (extra,)
+ assert not extra, f'got extra columns {extra!r}'
missing = want_cols - got_cols
- assert not missing, 'missing columns %r' % (missing,)
+ assert not missing, f'missing columns {missing!r}'
for colname, want_type in want_coltypes.items():
assert got_ds.columns[colname].type == want_type
assert list(got_ds.iterate(None, colname)) == [type2value[want_type]]
diff --git a/accelerator/test_methods/a_test_dataset_roundrobin.py b/accelerator/test_methods/a_test_dataset_roundrobin.py
index 1af0e457..9df3f788 100644
--- a/accelerator/test_methods/a_test_dataset_roundrobin.py
+++ b/accelerator/test_methods/a_test_dataset_roundrobin.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test sliceno="roundrobin" in dataset iteration.
'''
diff --git a/accelerator/test_methods/a_test_dataset_slice.py b/accelerator/test_methods/a_test_dataset_slice.py
index 3c369f74..f2f4b7ea 100644
--- a/accelerator/test_methods/a_test_dataset_slice.py
+++ b/accelerator/test_methods/a_test_dataset_slice.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test dataset iteration slicing.
'''
@@ -72,7 +68,7 @@ def get_chain(sliceno, slice, columns='a'):
def assert_fails(sliceno, slice, func=get, columns='a'):
try:
func(sliceno, slice, columns)
- raise Exception("Iterating with slice %r should have failed" % (slice,))
+ raise Exception(f"Iterating with slice {slice!r} should have failed")
except DatasetError:
pass
assert_fails(None, -101)
diff --git a/accelerator/test_methods/a_test_dataset_type_None.py b/accelerator/test_methods/a_test_dataset_type_None.py
index bb39b0dc..460e37c6 100644
--- a/accelerator/test_methods/a_test_dataset_type_None.py
+++ b/accelerator/test_methods/a_test_dataset_type_None.py
@@ -17,8 +17,6 @@
# #
############################################################################
-from __future__ import unicode_literals
-
description = r'''
Test the +None types in dataset_type.
@@ -106,4 +104,4 @@ def synthesis(job):
if ds_with_None:
want.insert(0, None)
got = list(typed.iterate(0, name))
- assert want == got, 'Column %r in %s has %r, should have %r' % (name, typed.quoted, got, want,)
+ assert want == got, f'Column {name!r} in {typed.quoted} has {got!r}, should have {want!r}'
diff --git a/accelerator/test_methods/a_test_dataset_type_chaining.py b/accelerator/test_methods/a_test_dataset_type_chaining.py
index 71ac4557..d5166435 100644
--- a/accelerator/test_methods/a_test_dataset_type_chaining.py
+++ b/accelerator/test_methods/a_test_dataset_type_chaining.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify the various dataset_type chaining options:
Building a chain with and without extra datasets in the source.
@@ -38,14 +34,14 @@ def verify(a, b):
for sliceno in range(slices):
a_data = list(Dataset.iterate_list(sliceno, col, a))
b_data = list(map(str, Dataset.iterate_list(sliceno, col, b)))
- assert a_data == b_data, '%r has different contents to %r in slice %d column %s' % (a, b, sliceno, col,)
+ assert a_data == b_data, f'{a!r} has different contents to {b!r} in slice {sliceno} column {col}'
def verify_sorted(a, b):
for col in 'abcd':
a_data = list(Dataset.iterate_list(None, col, a))
b_data = list(map(str, Dataset.iterate_list(None, col, b)))
a_data.sort()
b_data.sort()
- assert a_data == b_data, '%r has different contents to %r in column %s' % (a, b, col,)
+ assert a_data == b_data, f'{a!r} has different contents to {b!r} in column {col}'
def write(name, previous, low, high, filter=lambda ix: True):
dw = job.datasetwriter(
name=name,
@@ -55,7 +51,7 @@ def write(name, previous, low, high, filter=lambda ix: True):
w = dw.get_split_write()
for ix in range(low, high):
if filter(ix):
- w('%d' % (ix,), '%d.2' % (ix,), '%d%s' % (ix, '.5' if ix % 2 else ''), '[%d]' % (ix,))
+ w(f'{ix}', f'{ix}.2', f"{ix}{'.5' if ix % 2 else ''}", f'[{ix}]')
return dw.finish()
untyped_A = write('A', None, 0, 100)
untyped_B = write('B', untyped_A, 100, 1000)
diff --git a/accelerator/test_methods/a_test_dataset_type_corner_cases.py b/accelerator/test_methods/a_test_dataset_type_corner_cases.py
index d9cab8e9..033d52e7 100644
--- a/accelerator/test_methods/a_test_dataset_type_corner_cases.py
+++ b/accelerator/test_methods/a_test_dataset_type_corner_cases.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify various corner cases in dataset_type.
'''
@@ -37,7 +33,7 @@
from accelerator.dispatch import JobError
from accelerator.dataset import Dataset, DatasetWriter
from accelerator.dsutil import typed_writer
-from accelerator.compat import PY3, UTC
+from accelerator.compat import UTC
from accelerator.standard_methods import dataset_type
from accelerator import g
@@ -60,7 +56,7 @@ def verify(name, types, bytes_data, want, default=no_default, want_fail=False, a
uni_data = [v.decode('ascii') for v in bytes_data]
todo += [('ascii', uni_data,), ('unicode', uni_data,)]
for coltype, data in todo:
- dsname = '%s %s' % (name, coltype,)
+ dsname = f'{name} {coltype}'
_verify(dsname, types, data, coltype, want, default, want_fail, exact_types, kw)
def _verify(name, types, data, coltype, want, default, want_fail, exact_types, kw):
@@ -75,7 +71,7 @@ def check(got, fromstr, filtered=False):
if exact_types:
got = [(v, type(v).__name__) for v in got]
want1 = [(v, type(v).__name__) for v in want1]
- assert got == want1, 'Expected %r, got %r from %s.' % (want1, got, fromstr,)
+ assert got == want1, f'Expected {want1!r}, got {got!r} from {fromstr}.'
dw = DatasetWriter(name=name, columns={'data': coltype, 'extra': 'bytes'})
dw.set_slice(0)
for ix, v in enumerate(data):
@@ -93,26 +89,26 @@ def check(got, fromstr, filtered=False):
except JobError:
if want_fail:
continue
- raise Exception('Typing %r as %s failed.' % (bytes_ds, typ,))
- assert not want_fail, "Typing %r as %s should have failed, but didn't (%s)." % (bytes_ds, typ, jid)
+ raise Exception(f'Typing {bytes_ds!r} as {typ} failed.')
+ assert not want_fail, f"Typing {bytes_ds!r} as {typ} should have failed, but didn't ({jid})."
typed_ds = Dataset(jid)
got = list(typed_ds.iterate(0, 'data'))
- check(got, '%s (typed as %s from %r)' % (typed_ds, typ, bytes_ds,))
+ check(got, f'{typed_ds} (typed as {typ} from {bytes_ds!r})')
if opts.get('filter_bad'):
bad_ds = Dataset(jid, 'bad')
got_bad = list(bad_ds.iterate(0, 'data'))
- assert got_bad == [b'nah'], "%s should have had a single b'nah', but had %r" % (bad_ds, got_bad,)
+ assert got_bad == [b'nah'], f"{bad_ds} should have had a single b'nah', but had {got_bad!r}"
if 'filter_bad' not in opts and not callable(want):
opts['filter_bad'] = True
opts['column2type']['extra'] = 'int32_10'
jid = subjobs.build('dataset_type', datasets=dict(source=bytes_ds), options=opts)
typed_ds = Dataset(jid)
got = list(typed_ds.iterate(0, 'data'))
- check(got, '%s (typed as %s from %r with every other line skipped from filter_bad)' % (typed_ds, typ, bytes_ds,), True)
+ check(got, f'{typed_ds} (typed as {typ} from {bytes_ds!r} with every other line skipped from filter_bad)', True)
want_bad = [t for t in bytes_ds.iterate(0) if t[1] == b'skip']
bad_ds = Dataset(jid, 'bad')
got_bad = list(bad_ds.iterate(0))
- assert got_bad == want_bad, "Expected %r, got %r from %s" % (want_bad, got_bad, bad_ds,)
+ assert got_bad == want_bad, f"Expected {want_bad!r}, got {got_bad!r} from {bad_ds}"
used_type(typ)
def test_numbers():
@@ -129,22 +125,21 @@ def test_numbers():
(16, (b'1b', b'0x1b', b'\r001b',),),
( 0, (b'27', b'\r033', b'0x1b',),),
):
- types = ['%s_%d' % (typ, base,) for typ in ('int32', 'int64',)]
- verify('base %d' % (base,), types, values, [27, 27, 27], all_source_types=all_source_types)
+ types = [f'{typ}_{base}' for typ in ('int32', 'int64',)]
+ verify(f'base {base}', types, values, [27, 27, 27], all_source_types=all_source_types)
types = [typ + 'i' for typ in types]
if base == 10:
types += ['float32i', 'float64i']
values = [v + b'garbage' for v in values]
- verify('base %d i' % (base,), types, values, [27, 27, 27], all_source_types=all_source_types)
+ verify(f'base {base} i', types, values, [27, 27, 27], all_source_types=all_source_types)
all_source_types = False
- # python2 has both int and long, let's not check exact types there.
- verify('inty numbers', ['number', 'number:int'], [b'42', b'42.0', b'42.0000000', b'43.', b'.0'], [42, 42, 42, 43, 0], exact_types=PY3)
+ verify('inty numbers', ['number', 'number:int'], [b'42', b'42.0', b'42.0000000', b'43.', b'.0'], [42, 42, 42, 43, 0], exact_types=True)
if options.numeric_comma:
- verify('inty numbers numeric_comma', ['number', 'number:int'], [b'42', b'42,0', b'42,0000000', b'43,', b',0'], [42, 42, 42, 43, 0], numeric_comma=True, exact_types=PY3)
+ verify('inty numbers numeric_comma', ['number', 'number:int'], [b'42', b'42,0', b'42,0000000', b'43,', b',0'], [42, 42, 42, 43, 0], numeric_comma=True, exact_types=True)
- # Python 2 accepts 42L as an integer, python 3 doesn't. The number
+ # Python 3 does not accepts 42L as an integer. The number
# type falls back to python parsing, verify this works properly.
- verify('integer with L', ['number'], [b'42L'], [42], want_fail=PY3)
+ verify('integer with L', ['number'], [b'42L'], [42], want_fail=True)
# tests both that values outside the range are rejected
# and that None works as a default value.
@@ -166,7 +161,7 @@ def test_numbers():
verify('floatbool false', ['floatbool'], [b'0', b'-0', b'1', b'1004', b'0.00001', b'inf', b'-1', b' 0 ', b'0.00'], [False, False, True, True, True, True, True, False, False], exact_types=True)
verify('floatbool i', ['floatbooli'], [b'1 yes', b'0 no', b'0.00 also no', b'inf yes', b' 0.01y', b''], [True, False, False, True, True, False], exact_types=True)
def check_special(got, fromstr):
- msg = 'Expected [inf, -inf, nan, nan, nan, nan, inf], got %r from %s.' % (got, fromstr,)
+ msg = f'Expected [inf, -inf, nan, nan, nan, nan, inf], got {got!r} from {fromstr}.'
for ix, v in ((0, float('inf')), (1, float('-inf')), (-1, float('inf'))):
assert got[ix] == v, msg
for ix in range(2, 6):
@@ -296,18 +291,18 @@ def good(want, *a):
for value in a:
tests[pattern].append((value, want))
got = strptime(value, pattern)
- assert got == want, "Parsing %r as %r\n expected %s\n got %s" % (value, pattern, want, got,)
+ assert got == want, f"Parsing {value!r} as {pattern!r}\n expected {want}\n got {got}"
value += 'x'
got, remaining = strptime_i(value, pattern)
- assert got == want, "Parsing %r as %r\n expected %s\n got %s" % (value, pattern, want, got,)
- assert remaining == b'x', "Parsing %r as %r left %r unparsed, expected %r" % (value, pattern, remaining, b'x',)
+ assert got == want, f"Parsing {value!r} as {pattern!r}\n expected {want}\n got {got}"
+ assert remaining == b'x', f"Parsing {value!r} as {pattern!r} left {remaining!r} unparsed, expected {b'x'!r}"
def bad(*a):
for value in a:
tests[pattern].append((value, None))
try:
got = strptime(value, pattern)
- raise Exception("Parsing %r as %r gave %s, should have failed" % (value, pattern, got,))
+ raise Exception(f"Parsing {value!r} as {pattern!r} gave {got}, should have failed")
except ValueError:
pass
@@ -926,7 +921,7 @@ def check(type_as, fix):
for got, wrote, good in zip(got, wrote, good):
if good is not None:
good = fix(good)
- assert got == good, 'Typing %r as %r gave %r, expected %r' % (wrote, column2type[col], got, good,)
+ assert got == good, f'Typing {wrote!r} as {column2type[col]!r} gave {got!r}, expected {good!r}'
check('datetime', lambda dt: dt)
check('date', lambda dt: dt.date())
@@ -948,7 +943,7 @@ def check(type_as, fix):
('date', date),
('datetime', datetime),
):
- verify('nearly good %s YYYY-MM-DD' % (type_as,), ['%s:%s' % (type_as, pattern,)], [b'2019-02-29', b'1970-02-31', b'1980-06-31', b'1992-02-29'], [None, None, None, func(1992, 2, 29)], None, False)
+ verify(f'nearly good {type_as} YYYY-MM-DD', [f'{type_as}:{pattern}'], [b'2019-02-29', b'1970-02-31', b'1980-06-31', b'1992-02-29'], [None, None, None, func(1992, 2, 29)], None, False)
def test_filter_bad_across_types():
@@ -981,19 +976,17 @@ def test_filter_bad_across_types():
want_bad = [tuple(l[1:]) for l in data if not l[0]]
dw = DatasetWriter(name="filter bad across types", columns=columns, allow_missing_slices=True)
cols_to_check = ['int32_10', 'bytes', 'json', 'unicode:utf-8']
- if PY3:
- # z so it sorts last.
- dw.add('zpickle', 'pickle')
- cols_to_check.append('zpickle')
- for ix in range(len(data)):
- data[ix].append({ix})
+ # z so it sorts last.
+ dw.add('zpickle', 'pickle')
+ cols_to_check.append('zpickle')
+ for ix in range(len(data)):
+ data[ix].append({ix})
dw.set_slice(0)
want = []
def add_want(ix):
v = data[ix]
want.append((int(v[3]), v[1], json.loads(v[4]), v[6].decode('utf-8'),))
- if PY3:
- want[-1] = want[-1] + (v[7],)
+ want[-1] = want[-1] + (v[7],)
for ix, v in enumerate(data):
if v[0]:
add_want(ix)
@@ -1009,10 +1002,10 @@ def add_want(ix):
)
typed_ds = Dataset(jid)
got = list(typed_ds.iterate(0, cols_to_check))
- assert got == want, "Expected %r, got %r from %s (from %r%s)" % (want, got, typed_ds, source_ds, ' with defaults' if defaults else '')
+ assert got == want, f"Expected {want!r}, got {got!r} from {typed_ds} (from {source_ds!r}{' with defaults' if defaults else ''})"
bad_ds = Dataset(jid, 'bad')
got_bad = list(bad_ds.iterate(0, sorted(columns)))
- assert got_bad == want_bad, "Expected %r, got %r from %s (from %r%s)" % (want_bad, got_bad, bad_ds, source_ds, ' with defaults' if defaults else '')
+ assert got_bad == want_bad, f"Expected {want_bad!r}, got {got_bad!r} from {bad_ds} (from {source_ds!r}{' with defaults' if defaults else ''})"
# make more lines "ok" for the second lap
if not defaults:
want_bad.pop(0) # number:int
@@ -1220,7 +1213,7 @@ def test_column_discarding():
column2type=dict(a='ascii', c='ascii'),
discard_untyped=True,
).dataset()
- assert sorted(ac_implicit.columns) == ['a', 'c'], '%s: %r' % (ac_implicit, sorted(ac_implicit.columns),)
+ assert sorted(ac_implicit.columns) == ['a', 'c'], f'{ac_implicit}: {sorted(ac_implicit.columns)!r}'
assert list(ac_implicit.iterate(None)) == [('a', 'c',)], ac_implicit
# Discard b explicitly
@@ -1230,7 +1223,7 @@ def test_column_discarding():
column2type=dict(a='ascii', c='ascii'),
rename=dict(b=None),
).dataset()
- assert sorted(ac_explicit.columns) == ['a', 'c'], '%s: %r' % (ac_explicit, sorted(ac_explicit.columns),)
+ assert sorted(ac_explicit.columns) == ['a', 'c'], f'{ac_explicit}: {sorted(ac_explicit.columns)!r}'
assert list(ac_explicit.iterate(None)) == [('a', 'c',)], ac_explicit
# Discard c by overwriting it with b. Keep untyped b.
@@ -1240,7 +1233,7 @@ def test_column_discarding():
column2type=dict(a='ascii', c='ascii'),
rename=dict(b='c'),
).dataset()
- assert sorted(ac_bASc.columns) == ['a', 'b', 'c'], '%s: %r' % (ac_bASc, sorted(ac_bASc.columns),)
+ assert sorted(ac_bASc.columns) == ['a', 'b', 'c'], f'{ac_bASc}: {sorted(ac_bASc.columns)!r}'
assert list(ac_bASc.iterate(None)) == [('a', b'b', 'b',)], ac_bASc
# Discard c by overwriting it with b. Also type b as a different type.
@@ -1250,7 +1243,7 @@ def test_column_discarding():
column2type=dict(a='ascii', b='strbool', c='ascii'),
rename=dict(b='c'),
).dataset()
- assert sorted(abc_bASc.columns) == ['a', 'b', 'c'], '%s: %r' % (abc_bASc, sorted(abc_bASc.columns),)
+ assert sorted(abc_bASc.columns) == ['a', 'b', 'c'], f'{abc_bASc}: {sorted(abc_bASc.columns)!r}'
assert list(abc_bASc.iterate(None)) == [('a', True, 'b',)], abc_bASc
def test_rehash_with_empty_slices():
diff --git a/accelerator/test_methods/a_test_dataset_type_hashing.py b/accelerator/test_methods/a_test_dataset_type_hashing.py
index b951165e..eb914824 100644
--- a/accelerator/test_methods/a_test_dataset_type_hashing.py
+++ b/accelerator/test_methods/a_test_dataset_type_hashing.py
@@ -18,10 +18,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify that using dataset_type with a hashlabel gives the same result as
first typing and then rehashing for various hashlabel types, including
@@ -34,7 +30,7 @@
from itertools import cycle
from datetime import date, time, datetime
-from accelerator.compat import unicode, PY3
+from accelerator.compat import unicode
from accelerator import subjobs
from accelerator.extras import DotDict
from accelerator.dsutil import typed_writer
@@ -83,7 +79,7 @@ def synthesis(job, slices):
# Test various types for hashing and discarding of bad lines.
for hl in (None, 'a', 'b', 'c'):
- dw = job.datasetwriter(name='hashed on %s' % (hl,), columns={'a': 'unicode', 'b': 'unicode', 'c': 'unicode'}, hashlabel=hl)
+ dw = job.datasetwriter(name=f'hashed on {hl}', columns={'a': 'unicode', 'b': 'unicode', 'c': 'unicode'}, hashlabel=hl)
w = dw.get_split_write()
for ix in range(1000):
w(unicode(ix), '%d.%d' % (ix, ix % 5 == 0), ('{"a": %s}' if ix % 3 else '%d is bad') % (ix,))
@@ -99,10 +95,10 @@ def synthesis(job, slices):
dw = job.datasetwriter(name='more types')
cols = {
'floatbooli': cycle(['1.42 or so', '0 maybe', '1 (exactly)']),
- 'datetime:%Y%m%d %H:%M': ['2019%02d%02d 17:%02d' % (t % 12 + 1, t % 28 + 1, t % 60) for t in range(1000)],
- 'date:%Y%m%d': ['2019%02d%02d' % (t % 12 + 1, t % 28 + 1,) for t in range(1000)],
- 'time:%H:%M': ['%02d:%02d' % (t // 60, t % 60) for t in range(1000)],
- 'timei:%H:%M': ['%02d:%02d%c' % (t // 60, t % 60, chr(t % 26 + 65)) for t in range(1000)],
+ 'datetime:%Y%m%d %H:%M': [f'2019{t % 12 + 1:02}{t % 28 + 1:02} 17:{t % 60:02}' for t in range(1000)],
+ 'date:%Y%m%d': [f'2019{t % 12 + 1:02}{t % 28 + 1:02}' for t in range(1000)],
+ 'time:%H:%M': [f'{t // 60:02}:{t % 60:02}' for t in range(1000)],
+ 'timei:%H:%M': [f'{t // 60:02}:{t % 60:02}{chr(t % 26 + 65)}' for t in range(1000)],
}
gens = []
for coltype, gen in cols.items():
@@ -158,19 +154,17 @@ def synthesis(job, slices):
'time' : ('time', True),
'unicode' : ('unicode', True),
})
- if PY3:
- # name it with z so it's last during iteration.
- dw.add('zpickle', 'pickle', none_support=True)
+ # name it with z so it's last during iteration.
+ dw.add('zpickle', 'pickle', none_support=True)
write = dw.get_split_write()
data = {
'42': ('ascii string', True, b'bytes string', 1+2j, 2+3j, date(2019, 12, 11), datetime(2019, 12, 11, 20, 7, 21), 1.5, 0.00000001, 99, -11, {"a": "b"}, 1e100, time(20, 7, 21), 'unicode string'),
None: ( None, None, None, None, None, None, None, None, None, None, None, None, None, None, None),
'18': ('ASCII STRING', False, b'BYTES STRING', 3-4j, 4-5j, date(1868, 1, 3), datetime(1868, 1, 3, 13, 14, 5), 2.5, -0.0000001, 67, -99, [42, ".."], 5e100, time(13, 14, 5), 'UNICODE STRING'),
}
- if PY3:
- data['42'] += ([date(1, 2, 3), 'foo'],)
- data[None] += (None,)
- data['18'] += ({1, 2, 'c', b'bar'},)
+ data['42'] += ([date(1, 2, 3), 'foo'],)
+ data[None] += (None,)
+ data['18'] += ({1, 2, 'c', b'bar'},)
write('42', *data['42'])
write(None, *data[None])
write('18', *data['18'])
@@ -206,12 +200,12 @@ def synthesis(job, slices):
key = key.decode('ascii')
else:
key = unicode(key)
- assert data.get(key) == line[1:], "%s (hl %s) didn't have the right data for line %r" % (ds, hl, line[0],)
+ assert data.get(key) == line[1:], f"{ds} (hl {hl}) didn't have the right data for line {line[0]!r}"
hv = line[sorted(src_ds.columns).index(hl)]
- assert hl_hash(hv) % slices == sliceno, "%s (hl %s) didn't hash %r correctly" % (ds, hl, hv,)
- assert key not in seen, "%s (hl %s) repeated line %s" % (ds, hl, line[0],)
+ assert hl_hash(hv) % slices == sliceno, f"{ds} (hl {hl}) didn't hash {hv!r} correctly"
+ assert key not in seen, f"{ds} (hl {hl}) repeated line {line[0]}"
seen.add(key)
- assert seen == {'42', 'None', '18'}, "%s didn't have all lines (%r)" % (ds, seen,)
+ assert seen == {'42', 'None', '18'}, f"{ds} didn't have all lines ({seen!r})"
def test(src_ds, opts, expect_lines):
opts = DotDict(opts)
@@ -219,7 +213,7 @@ def rename(colname):
return opts.get('rename', {}).get(colname, colname)
cols = set(opts.column2type)
opts.discard_untyped = True
- msg = 'Testing with types %s' % (', '.join(v for k, v in sorted(opts.column2type.items())),)
+ msg = f"Testing with types {', '.join((v for k, v in sorted(opts.column2type.items())))}"
expect_hl = None
if src_ds.hashlabel and opts.column2type.get(src_ds.hashlabel) == 'json':
# json is not hashable, so we have to override the hashlabel to nothing in this case.
@@ -228,7 +222,7 @@ def rename(colname):
elif src_ds.hashlabel:
expect_hl = rename(src_ds.hashlabel)
if expect_hl in opts.column2type:
- msg += ' (hashed on %s)' % (opts.column2type[expect_hl],)
+ msg += f' (hashed on {opts.column2type[expect_hl]})'
else:
expect_hl = None
msg += ' (hashed on )'
@@ -258,7 +252,7 @@ def rename(colname):
# not hashable
continue
opts['hashlabel'] = hashlabel
- print('%s rehashed on %s' % (msg, opts.column2type[hashlabel],))
+ print(f'{msg} rehashed on {opts.column2type[hashlabel]}')
hashed_by_type = subjobs.build('dataset_type', options=opts, datasets=dict(source=src_ds)).dataset()
assert hashed_by_type.hashlabel == hashlabel, hashed_by_type
assert set(hashed_by_type.columns) == cols, hashed_by_type
diff --git a/accelerator/test_methods/a_test_dataset_type_minmax.py b/accelerator/test_methods/a_test_dataset_type_minmax.py
index c8d7d67c..7f91259e 100644
--- a/accelerator/test_methods/a_test_dataset_type_minmax.py
+++ b/accelerator/test_methods/a_test_dataset_type_minmax.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify that dataset_type gets .min and .max right.
'''
@@ -171,7 +167,7 @@ def synthesis(job):
minmax = (t_ds.columns['v'].min, t_ds.columns['v'].max)
if minmax != (None, None):
- raise Exception('Typing empty dataset as %s did not give minmax == None, gave %r' % (typ, minmax,))
+ raise Exception(f'Typing empty dataset as {typ} did not give minmax == None, gave {minmax!r}')
all_names = list(chain.from_iterable(groupdata[group].keys() for group in groups))
# just 1 and 2, so we don't make way too many
for num_groups in (1, 2,):
@@ -180,7 +176,7 @@ def synthesis(job):
t_ds = subjobs.build('dataset_type', column2type={'v': typ}, source=ds).dataset()
got_minmax = (t_ds.columns['v'].min, t_ds.columns['v'].max)
want_minmax = (mn, mx)
- chk_minmax(got_minmax, want_minmax, 'Typing %s as %s gave wrong minmax: expected %r, got %r (in %s)' % (ds.quoted, typ, want_minmax, got_minmax, t_ds.quoted,))
+ chk_minmax(got_minmax, want_minmax, f'Typing {ds.quoted} as {typ} gave wrong minmax: expected {want_minmax!r}, got {got_minmax!r} (in {t_ds.quoted})')
chk_minmax(got_minmax, (t_ds.min('v'), t_ds.max('v')), 'Dataset.min/max() broken on ' + t_ds)
# verify writing the same data normally also gives the correct result
dw = DatasetWriter(name='rewrite ' + t_ds, columns=t_ds.columns)
@@ -190,7 +186,7 @@ def synthesis(job):
re_ds = dw.finish()
got_minmax = (re_ds.columns['v'].min, re_ds.columns['v'].max)
want_minmax = (mn, mx)
- chk_minmax(got_minmax, want_minmax, 'Rewriting %s gave the wrong minmax: expected %r, got %r (in %s)' % (t_ds.quoted, want_minmax, got_minmax, re_ds.quoted,))
+ chk_minmax(got_minmax, want_minmax, f'Rewriting {t_ds.quoted} gave the wrong minmax: expected {want_minmax!r}, got {got_minmax!r} (in {re_ds.quoted})')
# make sure renaming doesn't mix anything up
dw = DatasetWriter(name='rename', columns={'a': 'ascii', 'b': 'ascii'})
@@ -210,5 +206,5 @@ def synthesis(job):
('int', (2, 3)),
):
got_minmax = (t_ds.columns[name].min, t_ds.columns[name].max)
- msg = 'Typing %s gave wrong minmax: expected %r, got %r (in %s)' % (ds.quoted, want_minmax, got_minmax, t_ds.quoted,)
+ msg = f'Typing {ds.quoted} gave wrong minmax: expected {want_minmax!r}, got {got_minmax!r} (in {t_ds.quoted})'
chk_minmax(got_minmax, want_minmax, msg)
diff --git a/accelerator/test_methods/a_test_dataset_unroundrobin.py b/accelerator/test_methods/a_test_dataset_unroundrobin.py
index 01b20218..ed1c8e22 100644
--- a/accelerator/test_methods/a_test_dataset_unroundrobin.py
+++ b/accelerator/test_methods/a_test_dataset_unroundrobin.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test that dataset_unroundrobin produces the correct order.
'''
@@ -42,7 +38,7 @@ def synthesis(job, slices):
dw = job.datasetwriter(name='rr', columns=dict(a='int32', b='unicode'))
for sliceno in range(slices):
dw.set_slice(sliceno)
- dw.write(sliceno, 'u %d' % (sliceno,))
+ dw.write(sliceno, f'u {sliceno}')
dw.write(sliceno, 'line 2')
dw.write(sliceno, 'line 3')
if sliceno == 0:
@@ -54,10 +50,10 @@ def want(a, b):
try:
got = next(it)
except StopIteration:
- raise Exception('missing lines in %s' % (ds_unrr,))
- assert got == (a, b), "Wanted %r, got %r from %s" % ((a, b,), got, ds_unrr,)
+ raise Exception(f'missing lines in {ds_unrr}')
+ assert got == (a, b), f"Wanted {(a, b)!r}, got {got!r} from {ds_unrr}"
for sliceno in range(slices):
- want(sliceno, 'u %d' % (sliceno,))
+ want(sliceno, f'u {sliceno}')
for sliceno in range(slices):
want(sliceno, 'line 2')
for sliceno in range(slices):
@@ -65,7 +61,7 @@ def want(a, b):
want(-1, 'line 4 just in slice 0')
try:
next(it)
- raise Exception("Extra lines in %s" % (ds_unrr,))
+ raise Exception(f"Extra lines in {ds_unrr}")
except StopIteration:
pass
@@ -74,7 +70,7 @@ def want(a, b):
imported = subjobs.build('csvimport', filename=exported.filename('unrr.csv'))
imported = subjobs.build('dataset_type', source=imported, column2type=dict(a='int32_10', b='ascii')).dataset()
for sliceno in range(slices):
- assert list(imported.iterate(sliceno)) == list(ds_rr.iterate(sliceno)), "%s did not match %s in slice %d, export or import does not match roundrobin expectations" % (imported, ds_rr, sliceno)
+ assert list(imported.iterate(sliceno)) == list(ds_rr.iterate(sliceno)), f"{imported} did not match {ds_rr} in slice {sliceno}, export or import does not match roundrobin expectations"
# Check that empty slices in the middle are not a problem.
dw = job.datasetwriter(name='empty slices', columns=dict(a='number'))
diff --git a/accelerator/test_methods/a_test_dataset_unroundrobin_trigger.py b/accelerator/test_methods/a_test_dataset_unroundrobin_trigger.py
index 2794bf6b..50606421 100644
--- a/accelerator/test_methods/a_test_dataset_unroundrobin_trigger.py
+++ b/accelerator/test_methods/a_test_dataset_unroundrobin_trigger.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test that dataset_unroundrobin with trigger_column produces the
correct order and switches slice at the expected point.
@@ -30,7 +26,7 @@
def assert_slice(ds, sliceno, want):
got = list(ds.iterate(sliceno))
- assert got == want, "slice %s in %s (from %s, trigger %s) gave\n\t%r,\nwanted\n\t%r" % (sliceno, ds, ds.job.params.datasets.source, ds.job.params.options.trigger_column, got, want,)
+ assert got == want, f"slice {sliceno} in {ds} (from {ds.job.params.datasets.source}, trigger {ds.job.params.options.trigger_column}) gave\n\t{got!r},\nwanted\n\t{want!r}"
def synthesis(job, slices):
unrr_values = [
diff --git a/accelerator/test_methods/a_test_datasetwriter.py b/accelerator/test_methods/a_test_datasetwriter.py
index 0fd34d22..fe601c63 100644
--- a/accelerator/test_methods/a_test_datasetwriter.py
+++ b/accelerator/test_methods/a_test_datasetwriter.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test DatasetWriter, exercising the different ways to create,
pass and populate the dataset.
diff --git a/accelerator/test_methods/a_test_datasetwriter_copy.py b/accelerator/test_methods/a_test_datasetwriter_copy.py
index 14912d7a..565b1d82 100644
--- a/accelerator/test_methods/a_test_datasetwriter_copy.py
+++ b/accelerator/test_methods/a_test_datasetwriter_copy.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test copy_mode in DatasetWriter, and three ways to specify the column
types (columns={}, add(n, Datasetcolums), .add(n, (t, none_support)).
@@ -66,6 +62,6 @@ def synthesis(job, slices, prepare_res):
ds = dw_nonetest_removed.finish()
for name, col in ds.columns.items():
if name == 'unicode':
- assert col.none_support, "%s:%s should have none_support" % (ds, name,)
+ assert col.none_support, f"{ds}:{name} should have none_support"
else:
- assert not col.none_support, "%s:%s shouldn't have none_support" % (ds, name,)
+ assert not col.none_support, f"{ds}:{name} shouldn't have none_support"
diff --git a/accelerator/test_methods/a_test_datasetwriter_default.py b/accelerator/test_methods/a_test_datasetwriter_default.py
index 0eb424d4..e0345e89 100644
--- a/accelerator/test_methods/a_test_datasetwriter_default.py
+++ b/accelerator/test_methods/a_test_datasetwriter_default.py
@@ -48,7 +48,7 @@ def synthesis(job):
try:
dw.add('data', t, default=bad_default)
dw.get_split_write()
- raise Exception('%s accepted %r as default value' % (t, bad_default,))
+ raise Exception(f'{t} accepted {bad_default!r} as default value')
except (TypeError, ValueError, OverflowError):
pass
dw.discard()
@@ -61,7 +61,7 @@ def synthesis(job):
ds = dw.finish()
want = [good_value, default_value, default_value]
got = list(ds.iterate(0, 'data'))
- assert got == want, '%s failed, wanted %r but got %r' % (ds.quoted, want, got,)
+ assert got == want, f'{ds.quoted} failed, wanted {want!r} but got {got!r}'
dw = job.datasetwriter(name=t + ' default=None', allow_missing_slices=True)
dw.add('data', t, default=None, none_support=True)
@@ -72,7 +72,7 @@ def synthesis(job):
ds = dw.finish()
want = [good_value, None, None]
got = list(ds.iterate(0, 'data'))
- assert got == want, '%s failed, wanted %r but got %r' % (ds.quoted, want, got,)
+ assert got == want, f'{ds.quoted} failed, wanted {want!r} but got {got!r}'
# make sure default=None hashes correctly
if t != 'json':
@@ -85,4 +85,4 @@ def synthesis(job):
ds = dw.finish()
want = [None, None, None]
got = list(ds.iterate(0, 'data'))
- assert got == want, '%s slice 0 failed, wanted %r but got %r' % (ds.quoted, want, got,)
+ assert got == want, f'{ds.quoted} slice 0 failed, wanted {want!r} but got {got!r}'
diff --git a/accelerator/test_methods/a_test_datasetwriter_missing_slices.py b/accelerator/test_methods/a_test_datasetwriter_missing_slices.py
index 292b91eb..c3235504 100644
--- a/accelerator/test_methods/a_test_datasetwriter_missing_slices.py
+++ b/accelerator/test_methods/a_test_datasetwriter_missing_slices.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test that missing set_slice is an error without allow_missing_slices
but not with.
diff --git a/accelerator/test_methods/a_test_datasetwriter_parent.py b/accelerator/test_methods/a_test_datasetwriter_parent.py
index 45fc1501..a33d9c01 100644
--- a/accelerator/test_methods/a_test_datasetwriter_parent.py
+++ b/accelerator/test_methods/a_test_datasetwriter_parent.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Exercise different valid and invalid parent/hashlabel combinations in
DatasetWriter.
diff --git a/accelerator/test_methods/a_test_datasetwriter_parsed.py b/accelerator/test_methods/a_test_datasetwriter_parsed.py
index e2e1c7e7..fa9fb2c0 100644
--- a/accelerator/test_methods/a_test_datasetwriter_parsed.py
+++ b/accelerator/test_methods/a_test_datasetwriter_parsed.py
@@ -38,7 +38,7 @@ def synthesis(job):
bad_value = object
else:
bad_value = 'not valid'
- name = '%s %s none_support' % (t, 'with' if none_support else 'without',)
+ name = f"{t} {'with' if none_support else 'without'} none_support"
dw = job.datasetwriter(name=name, allow_missing_slices=True)
dw.add('data', 'parsed:' + t, none_support=none_support)
dw.set_slice(0)
@@ -46,11 +46,11 @@ def synthesis(job):
dw.write(v)
dw.set_slice(1)
for v in parse_values:
- assert isinstance(v, str), 'oops: %r' % (v,)
+ assert isinstance(v, str), f'oops: {v!r}'
dw.write(v)
try:
dw.write(bad_value)
- raise Exception("parsed:%s accepted %r as a value" % (t, bad_value,))
+ raise Exception(f"parsed:{t} accepted {bad_value!r} as a value")
except (ValueError, TypeError):
pass
# json will of course accept None even without none_support
@@ -59,17 +59,17 @@ def synthesis(job):
try:
dw.write(None)
if not apparent_none_support:
- raise Exception('parsed:%s accepted None without none_support' % (t,))
+ raise Exception(f'parsed:{t} accepted None without none_support')
except (ValueError, TypeError):
if apparent_none_support:
- raise Exception('parsed:%s did not accept None despite none_support' % (t,))
+ raise Exception(f'parsed:{t} did not accept None despite none_support')
ds = dw.finish()
for sliceno, desc in enumerate(("normal values", "parseable values",)):
got = list(ds.iterate(sliceno, 'data'))
- assert got == values, "parsed:%s (%s) %s gave %r, wanted %r" % (t, ds.quoted, desc, got, values,)
+ assert got == values, f"parsed:{t} ({ds.quoted}) {desc} gave {got!r}, wanted {values!r}"
if apparent_none_support:
got = list(ds.iterate(2, 'data'))
- assert got == [None], "parsed:%s (%s) gave %r, wanted [None]" % (t, ds.quoted, got,)
+ assert got == [None], f"parsed:{t} ({ds.quoted}) gave {got!r}, wanted [None]"
dw = job.datasetwriter(name=name + ' with default', allow_missing_slices=True)
default = None if none_support else 42
dw.add('data', 'parsed:' + t, none_support=none_support, default=default)
@@ -78,4 +78,4 @@ def synthesis(job):
dw.write(bad_value)
ds = dw.finish()
got = list(ds.iterate(0, 'data'))
- assert got == [1, default], "parsed:%s with default=%s (%s) gave %r, wanted [1, %s]" % (t, default, ds.quoted, got, default)
+ assert got == [1, default], f"parsed:{t} with default={default} ({ds.quoted}) gave {got!r}, wanted [1, {default}]"
diff --git a/accelerator/test_methods/a_test_datasetwriter_verify.py b/accelerator/test_methods/a_test_datasetwriter_verify.py
index dc8f0350..844ee990 100644
--- a/accelerator/test_methods/a_test_datasetwriter_verify.py
+++ b/accelerator/test_methods/a_test_datasetwriter_verify.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify that each slice contains the expected data after test_datasetwriter.
'''
diff --git a/accelerator/test_methods/a_test_datetime.py b/accelerator/test_methods/a_test_datetime.py
index 3d0489a8..f2ecce27 100644
--- a/accelerator/test_methods/a_test_datetime.py
+++ b/accelerator/test_methods/a_test_datetime.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test the datetime types in options.
'''
diff --git a/accelerator/test_methods/a_test_hashlabel.py b/accelerator/test_methods/a_test_hashlabel.py
index 5501cc28..50045150 100644
--- a/accelerator/test_methods/a_test_hashlabel.py
+++ b/accelerator/test_methods/a_test_hashlabel.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test that hashlabel does what it says in both split_write and hashcheck.
Then test that rehashing gives the expected result, and that using the
@@ -110,7 +106,7 @@ def analysis(sliceno, prepare_res, params):
good = False
except Exception:
pass
- assert good, "%s allowed writing in wrong slice" % (fn,)
+ assert good, f"{fn} allowed writing in wrong slice"
# complex isn't sortable
def uncomplex(t):
@@ -161,7 +157,7 @@ def synthesis(prepare_res, params, job, slices):
data = [(ds, list(ds.iterate(sliceno))) for ds in hl2ds[hashlabel]]
good = data[0][1]
for name, got in data:
- assert got == good, "%s doesn't match %s in slice %d" % (data[0][0], name, sliceno,)
+ assert got == good, f"{data[0][0]} doesn't match {name} in slice {sliceno}"
# Verify that both up and down hashed on the expected column
hash = typed_writer("complex64").hash
@@ -169,7 +165,7 @@ def synthesis(prepare_res, params, job, slices):
ds = all_ds[colname + "_checked"]
for sliceno in range(slices):
for value in ds.iterate(sliceno, colname):
- assert hash(value) % slices == sliceno, "Bad hashing on %s in slice %d" % (colname, sliceno,)
+ assert hash(value) % slices == sliceno, f"Bad hashing on {colname} in slice {sliceno}"
# Verify that up and down are not the same, to catch hashing
# not actually hashing.
@@ -181,9 +177,9 @@ def synthesis(prepare_res, params, job, slices):
):
up = cleanup(all_ds[up_name].iterate(None))
down = cleanup(all_ds[down_name].iterate(None))
- assert up != down, "Hashlabel did not change slice distribution (%s vs %s)" % (up_name, down_name)
+ assert up != down, f"Hashlabel did not change slice distribution ({up_name} vs {down_name})"
# And check that the data is still the same.
- assert sorted(up) == sorted(down) == all_data, "Hashed datasets have wrong data (%s vs %s)" % (up_name, down_name)
+ assert sorted(up) == sorted(down) == all_data, f"Hashed datasets have wrong data ({up_name} vs {down_name})"
# Verify that rehashing works.
# (Can't use sliceno None, because that won't rehash, and even if it did
@@ -197,7 +193,7 @@ def test_rehash(want_ds, chk_ds_lst):
assert chk_ds.hashlabel != want_ds.hashlabel
got = chk_ds.iterate(sliceno, hashlabel=want_ds.hashlabel, rehash=True)
got = sorted(cleanup(got))
- assert want == got, "Rehashing is broken for %s (slice %d of %s)" % (chk_ds.columns[want_ds.hashlabel].type, sliceno, chk_ds,)
+ assert want == got, f"Rehashing is broken for {chk_ds.columns[want_ds.hashlabel].type} (slice {sliceno} of {chk_ds})"
test_rehash("up_checked", hl2ds[None] + hl2ds["down"])
test_rehash("down_checked", hl2ds[None] + hl2ds["up"])
test_rehash("up_datetime", [all_ds["down_time"]])
@@ -234,4 +230,4 @@ def test_rehash(want_ds, chk_ds_lst):
values = [(ds, list(ds.iterate(sliceno, "value"))) for ds in float_ds_lst]
want_ds, want = values.pop()
for ds, got in values:
- assert got == want, "%s did not match %s in slice %d" % (ds, want_ds, sliceno,)
+ assert got == want, f"{ds} did not match {want_ds} in slice {sliceno}"
diff --git a/accelerator/test_methods/a_test_hashpart.py b/accelerator/test_methods/a_test_hashpart.py
index c05246e3..94a0d473 100644
--- a/accelerator/test_methods/a_test_hashpart.py
+++ b/accelerator/test_methods/a_test_hashpart.py
@@ -18,10 +18,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify the dataset_hashpart method with various options.
'''
@@ -87,12 +83,12 @@ def verify(slices, data, source, previous=None, **options):
for slice in range(slices):
for row in ds.iterate_chain(slice, names):
row = dict(zip(names, row))
- assert h(row[hl]) % slices == slice, "row %r is incorrectly in slice %d in %s" % (row, slice, ds)
+ assert h(row[hl]) % slices == slice, f"row {row!r} is incorrectly in slice {slice} in {ds}"
want = good[row[hl]]
- assert row == want, '%s (rehashed from %s) did not contain the right data for "%s".\nWanted\n%r\ngot\n%r' % (ds, source, hl, want, row)
+ assert row == want, f'{ds} (rehashed from {source}) did not contain the right data for "{hl}".\nWanted\n{want!r}\ngot\n{row!r}'
want_lines = len(data)
got_lines = ds.chain().lines()
- assert got_lines == want_lines, '%s (rehashed from %s) had %d lines, should have had %d' % (ds, source, got_lines, want_lines,)
+ assert got_lines == want_lines, f'{ds} (rehashed from {source}) had {got_lines} lines, should have had {want_lines}'
return ds
def verify_empty(source, previous=None, **options):
@@ -103,8 +99,8 @@ def verify_empty(source, previous=None, **options):
)
ds = Dataset(jid)
chain = ds.chain_within_job()
- assert list(chain.iterate(None)) == [], "source=%s previous=%s did not produce empty dataset in %s" % (source, previous, ds,)
- assert chain[0].previous == previous, "Empty %s should have had previous=%s, but had %s" % (ds, previous, chain[0].previous,)
+ assert list(chain.iterate(None)) == [], f"source={source} previous={previous} did not produce empty dataset in {ds}"
+ assert chain[0].previous == previous, f"Empty {ds} should have had previous={previous}, but had {chain[0].previous}"
def synthesis(params):
ds = write(data)
@@ -121,7 +117,7 @@ def synthesis(params):
# normal chaining
a = verify(params.slices, data, ds, hashlabel="date")
b = verify(params.slices, data + bonus_data, bonus_ds, hashlabel="date", previous=a)
- assert b.chain() == [a, b], "chain of %s is not [%s, %s] as expected" % (b, a, b)
+ assert b.chain() == [a, b], f"chain of {b} is not [{a}, {b}] as expected"
# chain_slices sparseness
empty = write([], name="empty")
verify_empty(empty, hashlabel="date")
@@ -135,7 +131,7 @@ def synthesis(params):
dw.write_dict(data[0])
ds = verify(params.slices, [data[0], data[0]], dw.finish(), hashlabel="date", chain_slices=True)
got_slices = len(ds.chain())
- assert got_slices == params.slices, "%s (built with chain_slices=True) has %d datasets in chain, expected %d." % (ds, got_slices, params.slices)
+ assert got_slices == params.slices, f"{ds} (built with chain_slices=True) has {got_slices} datasets in chain, expected {params.slices}."
# test varying types and available columns over the chain (including the hashlabel type)
v1 = write([{'a': '101', 'b': 201 }], columns={'a': 'ascii', 'b': 'int32'}, name='varying1')
diff --git a/accelerator/test_methods/a_test_job_save.py b/accelerator/test_methods/a_test_job_save.py
index 736afbda..b7bec41d 100644
--- a/accelerator/test_methods/a_test_job_save.py
+++ b/accelerator/test_methods/a_test_job_save.py
@@ -25,17 +25,17 @@
def save(job, name, sliceno):
- p = job.save('contents of %s %s' % (name, sliceno,), name + '.pickle', sliceno=sliceno)
+ p = job.save(f'contents of {name} {sliceno}', name + '.pickle', sliceno=sliceno)
j = job.json_save({name: sliceno}, name + '.json', sliceno=sliceno)
name += '_path'
- p_path = job.save('contents of %s %s' % (name, sliceno,), Path(name + '.pickle'), sliceno=sliceno)
+ p_path = job.save(f'contents of {name} {sliceno}', Path(name + '.pickle'), sliceno=sliceno)
j_path = job.json_save({name: sliceno}, Path(name + '.json'), sliceno=sliceno)
return p, j, p_path, j_path
def check(job, name, sliceno, p, j, p_path, j_path):
- assert p.load() == job.load(Path(p.filename)) == 'contents of %s %s' % (name, sliceno,)
+ assert p.load() == job.load(Path(p.filename)) == f'contents of {name} {sliceno}'
assert j.load() == job.json_load(Path(j.filename)) == {name: sliceno}
- assert p_path.load() == job.load(Path(p_path.filename)) == 'contents of %s_path %s' % (name, sliceno,)
+ assert p_path.load() == job.load(Path(p_path.filename)) == f'contents of {name}_path {sliceno}'
assert j_path.load() == job.json_load(Path(j_path.filename)) == {name + '_path': sliceno}
for obj, filename in [
# Use Path() for files saved with strings and strings for files saved with Path.
diff --git a/accelerator/test_methods/a_test_job_save_background.py b/accelerator/test_methods/a_test_job_save_background.py
index 76603b7d..5b5f734b 100644
--- a/accelerator/test_methods/a_test_job_save_background.py
+++ b/accelerator/test_methods/a_test_job_save_background.py
@@ -37,7 +37,6 @@
options = {'sleeptime': 0.5}
from accelerator.compat import monotonic
-from accelerator.compat import unicode
import time
# This pickles as a string, but slowly.
@@ -51,7 +50,7 @@ def __init__(self, text, sleeptime):
def __reduce__(self):
time.sleep(self.sleeptime)
- return unicode, (unicode(self.text),)
+ return str, (str(self.text),)
# This is a True boolean that takes a long time to evaluate.
# It's a hack to make json encoding slow.
@@ -73,14 +72,14 @@ def __bool__(self):
def save(job, name, sliceno):
before = monotonic()
- p = job.save('contents of %s %s' % (name, sliceno,), name + '.pickle', sliceno=sliceno)
+ p = job.save(f'contents of {name} {sliceno}', name + '.pickle', sliceno=sliceno)
j = job.json_save({name: sliceno}, name + '.json', sliceno=sliceno)
name = 'background ' + name
- bp = job.save(SlowToPickle('contents of %s %s' % (name, sliceno,), options.sleeptime), name + '.pickle', sliceno=sliceno, background=True)
+ bp = job.save(SlowToPickle(f'contents of {name} {sliceno}', options.sleeptime), name + '.pickle', sliceno=sliceno, background=True)
bj = job.json_save({name: sliceno}, name + '.json', sliceno=sliceno, sort_keys=SlowTrue(options.sleeptime), background=True)
save_time = monotonic() - before
max_time = options.sleeptime * 2 # two slow files
- assert save_time < max_time, "Saving took %s seconds, should have been less than %s" % (save_time, max_time,)
+ assert save_time < max_time, f"Saving took {save_time} seconds, should have been less than {max_time}"
return p, j, bp, bj
def check(job, name, sliceno, p, j, bp, bj, do_background=True, do_wait=False):
@@ -92,7 +91,7 @@ def check(job, name, sliceno, p, j, bp, bj, do_background=True, do_wait=False):
# Do explicit waiting sometimes
p.wait()
j.wait()
- assert p.load() == 'contents of %s %s' % (name, sliceno,)
+ assert p.load() == f'contents of {name} {sliceno}'
assert j.load() == {name: sliceno}
for obj, filename in [(p, name + '.pickle'), (j, name + '.json')]:
path = job.filename(filename, sliceno=sliceno)
@@ -111,12 +110,12 @@ def prepare(job):
p = job.save(SlowToPickle('', checktime), 'test.pickle', background=True)
p.wait()
pickle_time = monotonic() - before
- assert pickle_time > checktime, "Saving a slow pickle took %s seconds, should have taken more than %s" % (pickle_time, checktime,)
+ assert pickle_time > checktime, f"Saving a slow pickle took {pickle_time} seconds, should have taken more than {checktime}"
before = monotonic()
j = job.json_save({}, 'test.json', sort_keys=SlowTrue(checktime), background=True)
j.wait()
json_time = monotonic() - before
- assert json_time > checktime, "Saving a slow json took %s seconds, should have taken more than %s" % (json_time, checktime,)
+ assert json_time > checktime, f"Saving a slow json took {json_time} seconds, should have taken more than {checktime}"
return res
diff --git a/accelerator/test_methods/a_test_jobchain.py b/accelerator/test_methods/a_test_jobchain.py
index a1b42dbf..e273c56b 100644
--- a/accelerator/test_methods/a_test_jobchain.py
+++ b/accelerator/test_methods/a_test_jobchain.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test Job.chain
'''
diff --git a/accelerator/test_methods/a_test_jobwithfile.py b/accelerator/test_methods/a_test_jobwithfile.py
index e983ebcb..d090e84f 100644
--- a/accelerator/test_methods/a_test_jobwithfile.py
+++ b/accelerator/test_methods/a_test_jobwithfile.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test JobWithFile file loading.
Pickle and json, sliced and unsliced.
diff --git a/accelerator/test_methods/a_test_json.py b/accelerator/test_methods/a_test_json.py
index 90854263..fdabf9fc 100644
--- a/accelerator/test_methods/a_test_json.py
+++ b/accelerator/test_methods/a_test_json.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify a few corner cases in the json functions in extras.
'''
@@ -29,7 +25,6 @@
from itertools import permutations
from accelerator.extras import json_save, json_load, json_encode
-from accelerator.compat import PY2, PY3
def test(name, input, want_obj, want_bytes, **kw):
json_save(input, name, **kw)
@@ -39,16 +34,15 @@ def test(name, input, want_obj, want_bytes, **kw):
got_bytes_raw = got_bytes_raw[:-1]
as_str = json_encode(input, as_str=True, **kw)
as_bytes = json_encode(input, as_str=False, **kw)
- assert isinstance(as_str, str) and isinstance(as_bytes, bytes), "json_encode returns the wrong types: %s %s" % (type(as_str), type(as_bytes),)
+ assert isinstance(as_str, str) and isinstance(as_bytes, bytes), f"json_encode returns the wrong types: {type(as_str)} {type(as_bytes)}"
assert as_bytes == got_bytes_raw, "json_save doesn't save the same thing json_encode returns for " + name
- if PY3:
- as_str = as_str.encode("utf-8")
+ as_str = as_str.encode("utf-8")
assert as_bytes == as_str, "json_encode doesn't return the same data for as_str=True and False"
got_obj = json_load(name)
- assert want_obj == got_obj, "%s roundtrips wrong (wanted %r, got %r)" % (name, want_obj, got_obj)
+ assert want_obj == got_obj, f"{name} roundtrips wrong (wanted {want_obj!r}, got {got_obj!r})"
with open(name, "rb") as fh:
got_bytes_fuzzy = b"".join(line.strip() for line in fh)
- assert want_bytes == got_bytes_fuzzy, "%s wrong on disk (but decoded right)" % (name,)
+ assert want_bytes == got_bytes_fuzzy, f"{name} wrong on disk (but decoded right)"
def synthesis():
test(
@@ -79,8 +73,6 @@ def synthesis():
)
unicode_want = u"bl\xe4"
- if PY2:
- unicode_want = unicode_want.encode("utf-8")
test(
"unicode.json",
u"bl\xe4",
@@ -93,7 +85,7 @@ def synthesis():
fh.write(b'"bl\xc3\xa4"')
assert json_load("utf-8.json") == unicode_want
- # This is supposed to work on PY2, but not PY3.
+ # This not supposed to work on PY3.
try:
test(
"string encoding.json",
@@ -101,13 +93,9 @@ def synthesis():
[b"\xc3\xa4", b"\xc3\xa4", [b"\xc3\xa4", {b"\xc3\xa4": b"\xc3\xa4",},],],
b'["\\u00e4","\\u00e4",["\\u00e4",{"\\u00e4": "\\u00e4"}]]',
)
- string_encoding_ok = True
+ assert False, "Bytes are not supposed to work in json_encode on PY3"
except TypeError:
- string_encoding_ok = False
- if PY2:
- assert string_encoding_ok, "Bytes are supposed to work in json_encode on PY2"
- else:
- assert not string_encoding_ok, "Bytes are not supposed to work in json_encode on PY3"
+ pass
# 720 permutations might be a bit much, but at least it's unlikely to
# miss ordering problems.
@@ -117,10 +105,10 @@ def synthesis():
s = "{"
for k, v in pairs:
d[k] = v
- s += '"%s": %s,' % (k, v)
+ s += f'"{k}": {v},'
s = (s[:-1] + "}").encode("ascii")
if not sorted_s:
sorted_s = s
sorted_d = d
- test("ordered%d.json" % (ix,), d, d, s, sort_keys=False)
- test("sorted%d.json" % (ix,), d, sorted_d, sorted_s, sort_keys=True)
+ test(f"ordered{ix}.json", d, d, s, sort_keys=False)
+ test(f"sorted{ix}.json", d, sorted_d, sorted_s, sort_keys=True)
diff --git a/accelerator/test_methods/a_test_nan.py b/accelerator/test_methods/a_test_nan.py
index 86b215bb..efb4f17c 100644
--- a/accelerator/test_methods/a_test_nan.py
+++ b/accelerator/test_methods/a_test_nan.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
import gzip
import math
import os
@@ -72,7 +68,7 @@ def test_encoding_and_hash(typ, nan, *want):
with gzip.open('tmp', 'rb') as fh:
got_bytes = fh.read()
os.remove('tmp')
- assert got_bytes in want, 'bad NaN representation in %s: got %r, wanted something in %r' % (typ, got_bytes, want,)
+ assert got_bytes in want, f'bad NaN representation in {typ}: got {got_bytes!r}, wanted something in {want!r}'
# Test that they either encode as themselves or as one of the normal NaNs,
# and all hash the same as the standard float64 NaN.
@@ -108,18 +104,18 @@ def mk_dws(typ):
h = typed_writer(typ).hash
want_h = (want_hash_complex if typ.startswith('complex') else want_hash_float)
for ix, v in enumerate(values):
- assert want_h == h(v), 'value index %d did not hash correctly for type %s' % (ix, typ,)
+ assert want_h == h(v), f'value index {ix} did not hash correctly for type {typ}'
w_u(str(ix), v)
w_h(str(ix), v)
ds_h = dw_h.finish()
- assert set(ds_h.lines) == {0, len(values)}, 'Not all NaNs ended up in the same slice in %s' % (ds_h.quoted,)
+ assert set(ds_h.lines) == {0, len(values)}, f'Not all NaNs ended up in the same slice in {ds_h.quoted}'
expect_lines = ds_h.lines
ds_u = dw_u.finish()
ds = build('dataset_hashpart', source=ds_u, hashlabel='nan').dataset()
- assert set(ds.lines) == {0, len(values)}, 'Not all NaNs ended up in the same slice in %s (dataset_hashpart from %s)' % (ds.quoted, ds_u.quoted,)
- assert expect_lines == ds.lines, 'dataset_hashpart (%s) disagrees with datasetwriter (%s) about NaN slicing' % (ds.quoted, ds_h.quoted,)
+ assert set(ds.lines) == {0, len(values)}, f'Not all NaNs ended up in the same slice in {ds.quoted} (dataset_hashpart from {ds_u.quoted})'
+ assert expect_lines == ds.lines, f'dataset_hashpart ({ds.quoted}) disagrees with datasetwriter ({ds_h.quoted}) about NaN slicing'
ds = build('dataset_type', source=ds_u, hashlabel='nan', column2type={'ix': 'number'}).dataset()
- assert set(ds.lines) == {0, len(values)}, 'Not all NaNs ended up in the same slice in %s (dataset_type from %s)' % (ds.quoted, ds_u.quoted,)
- assert expect_lines == ds.lines, 'dataset_type (%s) disagrees with datasetwriter (%s) about NaN slicing' % (ds.quoted, ds_h.quoted,)
+ assert set(ds.lines) == {0, len(values)}, f'Not all NaNs ended up in the same slice in {ds.quoted} (dataset_type from {ds_u.quoted})'
+ assert expect_lines == ds.lines, f'dataset_type ({ds.quoted}) disagrees with datasetwriter ({ds_h.quoted}) about NaN slicing'
rehash_lines = [len(list(ds_u.iterate(sliceno, rehash=True, hashlabel='nan'))) for sliceno in range(slices)]
- assert expect_lines == rehash_lines, 'ds.iterate(rehash=True) of %s disagrees with datasetwriter hashing (%s) about NaN slicing' % (ds_u.quoted, ds_h.quoted,)
+ assert expect_lines == rehash_lines, f'ds.iterate(rehash=True) of {ds_u.quoted} disagrees with datasetwriter hashing ({ds_h.quoted}) about NaN slicing'
diff --git a/accelerator/test_methods/a_test_number.py b/accelerator/test_methods/a_test_number.py
index b77b1443..482b8631 100644
--- a/accelerator/test_methods/a_test_number.py
+++ b/accelerator/test_methods/a_test_number.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
import gzip
from random import randint
import struct
@@ -102,7 +98,7 @@ def encode_as_big_number(num):
for v in values:
value = v[0]
write(value)
- csv_fh.write('%s\n' % (value,))
+ csv_fh.write(f'{value}\n')
if len(v) == 1:
want_bytes = encode_as_big_number(v[0])
else:
@@ -112,7 +108,7 @@ def encode_as_big_number(num):
w.write(value)
with gzip.open('tmp', 'rb') as fh:
got_bytes = fh.read()
- assert want_bytes == got_bytes, "%r gave %r, wanted %r" % (value, got_bytes, want_bytes,)
+ assert want_bytes == got_bytes, f"{value!r} gave {got_bytes!r}, wanted {want_bytes!r}"
# Make sure we get the same representation through a dataset.
# Assumes that the column is merged (a single file for all slices).
@@ -131,7 +127,7 @@ def encode_as_big_number(num):
ds_typed = jid.dataset()
with gzip.open(ds_typed.column_filename('num'), 'rb') as fh:
got_bytes = fh.read()
- assert want_bytes == got_bytes, "csvimport + dataset_type (%s) gave different bytes" % (jid,)
+ assert want_bytes == got_bytes, f"csvimport + dataset_type ({jid}) gave different bytes"
# Also test the hashing is as expected, both in DatasetWriter, dataset_type and dataset_hashpart.
def hash_num(num):
@@ -179,4 +175,4 @@ def endianfix(v):
('dataset_hashpart', ds_hashed),
('dataset_type', ds_typehashed),
]:
- assert set(ds.iterate(sliceno, 'num')) == per_slice[sliceno], 'wrong in slice %d in %s (hashed by %s)' % (sliceno, ds, hashname)
+ assert set(ds.iterate(sliceno, 'num')) == per_slice[sliceno], f'wrong in slice {sliceno} in {ds} (hashed by {hashname})'
diff --git a/accelerator/test_methods/a_test_optionenum.py b/accelerator/test_methods/a_test_optionenum.py
index 28f900e3..fd6eb816 100644
--- a/accelerator/test_methods/a_test_optionenum.py
+++ b/accelerator/test_methods/a_test_optionenum.py
@@ -18,16 +18,11 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test OptionEnum construction and enforcement.
'''
from accelerator.extras import OptionEnum
-from accelerator.compat import PY2
from accelerator import subjobs
from accelerator import blob
@@ -61,7 +56,7 @@ def check(**options):
want_res.update(pass_options)
jid = subjobs.build("test_optionenum", options=pass_options)
res = blob.load(jobid=jid)
- assert res == want_res, "%r != %r from %r" % (res, want_res, options,)
+ assert res == want_res, f"{res!r} != {want_res!r} from {options!r}"
def check_unbuildable(**options):
try:
@@ -70,18 +65,14 @@ def check_unbuildable(**options):
if e.args[0].startswith("Submit failed"):
return
raise
- raise Exception("Building with options = %r should have failed but didn't" % (options,))
+ raise Exception(f"Building with options = {options!r} should have failed but didn't")
def synthesis():
if options.inner:
return dict(options)
check(abcd="a", efgh="h")
check(ijkl="j", efgh="e")
- if PY2:
- # Passing "ä" is fine on PY2 too, but we get UTF-8 byte strings back.
- check(uni=b"\xc3\xa4")
- else:
- check(uni="ä")
+ check(uni="ä")
check(ijkl=None, dict=dict(foo="j"))
check(mnSTARop="p", qSTARrSTARst="qwe", ijkl="k")
check(mnSTARop="nah", qSTARrSTARst="really good value\n")
diff --git a/accelerator/test_methods/a_test_output.py b/accelerator/test_methods/a_test_output.py
index 735bb4a1..afb992b5 100644
--- a/accelerator/test_methods/a_test_output.py
+++ b/accelerator/test_methods/a_test_output.py
@@ -18,10 +18,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify that the output from methods is captured correctly for all valid
combinations of prepare/analysis/synthesis, both in OUTPUT dir and in
@@ -43,13 +39,13 @@ def test(params, p=False, a=False, s=False):
cookie = randint(10000, 99999)
if p:
name += 'p'
- opts['p'] = "Some words\nfrom prepare\nwith %d in them." % (cookie,)
+ opts['p'] = f"Some words\nfrom prepare\nwith {cookie} in them."
if a:
name += 'a'
opts['a'] = "A few words\nfrom analysis(%%d)\nwith the cookie %d in them." % (cookie,)
if s:
name += 's'
- opts['s'] = "Words\nfrom synthesis\ncookie is %d." % (cookie,)
+ opts['s'] = f"Words\nfrom synthesis\ncookie is {cookie}."
jid = subjobs.build(name, options=opts)
d = jid.filename('OUTPUT/')
chked = set()
@@ -65,8 +61,8 @@ def chk(part):
with open(d + part, 'r') as fh:
got = fh.read().replace('\r\n', '\n')
want = prefix + '\n' + data + '\n'
- assert got == prefix + '\n' + data + '\n', "%s produced %r in %s, expected %r" % (jid, got, part, want,)
- assert output == got, 'job.output disagrees with manual file reading for %s in %s. %r != %r' % (part, jid, output, got,)
+ assert got == prefix + '\n' + data + '\n', f"{jid} produced {got!r} in {part}, expected {want!r}"
+ assert output == got, f'job.output disagrees with manual file reading for {part} in {jid}. {output!r} != {got!r}'
all.append(got)
if p:
chk('prepare')
@@ -76,10 +72,10 @@ def chk(part):
if s:
chk('synthesis')
unchked = set(os.listdir(d)) - chked
- assert not unchked, "Unexpected OUTPUT files from %s: %r" % (jid, unchked,)
+ assert not unchked, f"Unexpected OUTPUT files from {jid}: {unchked!r}"
output = jid.output()
got = ''.join(all)
- assert output == got, 'job.output disagrees with manual file reading for in %s. %r != %r' % (jid, output, got,)
+ assert output == got, f'job.output disagrees with manual file reading for in {jid}. {output!r} != {got!r}'
def synthesis(params):
test(params, s=True)
@@ -110,7 +106,7 @@ def verify(want):
# it might not have reached the server yet
timeout += 0.01
# we've given it 3 seconds, it's not going to happen.
- raise Exception("Wanted to see tail output of %r, but saw %r" % (want, got,))
+ raise Exception(f"Wanted to see tail output of {want!r}, but saw {got!r}")
print(opts.prefix, file=sys.stderr)
verify(opts.prefix + '\n')
if isinstance(sliceno, int):
diff --git a/accelerator/test_methods/a_test_output_on_error.py b/accelerator/test_methods/a_test_output_on_error.py
index 148d4414..831560b4 100644
--- a/accelerator/test_methods/a_test_output_on_error.py
+++ b/accelerator/test_methods/a_test_output_on_error.py
@@ -30,7 +30,7 @@
def synthesis():
- lines = ['printing a bunch of lines, this is line %d.' % (n,) for n in range(150)]
+ lines = [f'printing a bunch of lines, this is line {n}.' for n in range(150)]
if options.inner:
for s in lines:
print(s)
@@ -51,6 +51,6 @@ def synthesis():
return
# not yet, wait a little (total of 30s)
if attempt > 1:
- print('Output from %s has not appeared yet, waiting more (%d).' % (job, attempt,))
+ print(f'Output from {job} has not appeared yet, waiting more ({attempt}).')
sleep(attempt / 10.0)
- raise Exception('Not all output from %s was saved in OUTPUT' % (job,))
+ raise Exception(f'Not all output from {job} was saved in OUTPUT')
diff --git a/accelerator/test_methods/a_test_rechain.py b/accelerator/test_methods/a_test_rechain.py
index c052ba71..1f7429ee 100644
--- a/accelerator/test_methods/a_test_rechain.py
+++ b/accelerator/test_methods/a_test_rechain.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test re-using datasets (from test_selfchain) in new
chains, and verify that the old chain still works.
@@ -38,7 +34,7 @@ def synthesis(job):
# build a local abf chain
prev = None
for ix, ds in enumerate(manual_abf):
- name = "abf%d" % (ix,)
+ name = f"abf{ix}"
prev = ds.link_to_here(name, override_previous=prev)
manual_abf_data = list(Dataset.iterate_list(None, None, manual_abf))
local_abf_data = list(Dataset(job, "abf2").iterate_chain(None, None))
@@ -56,7 +52,7 @@ def synthesis(job):
while going:
if prev and "cache" in prev._data:
going = False
- name = "longchain%d" % (ix,)
+ name = f"longchain{ix}"
dw = DatasetWriter(name=name, previous=prev)
dw.add("ix", "number")
dw.get_split_write()(ix)
diff --git a/accelerator/test_methods/a_test_register_file.py b/accelerator/test_methods/a_test_register_file.py
index a52ce5e0..025f1c37 100644
--- a/accelerator/test_methods/a_test_register_file.py
+++ b/accelerator/test_methods/a_test_register_file.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
from accelerator import subjobs
description = r'''
@@ -38,11 +34,11 @@ def test(method, *want_files):
if want_files != got_files:
extra_files = got_files - want_files
missing_files = want_files - got_files
- msg = "Got the wrong files from %s: %r" % (method, got_files,)
+ msg = f"Got the wrong files from {method}: {got_files!r}"
if extra_files:
- msg += ", did not expect %r" % (extra_files,)
+ msg += f", did not expect {extra_files!r}"
if missing_files:
- msg += ", also wanted %r" % (missing_files,)
+ msg += f", also wanted {missing_files!r}"
raise Exception(msg)
test('test_register_file_auto', 'analysis slice 0.txt', 'synthesis file.txt', 'result.pickle')
diff --git a/accelerator/test_methods/a_test_register_file_auto.py b/accelerator/test_methods/a_test_register_file_auto.py
index a6ca4c23..9d1fceb1 100644
--- a/accelerator/test_methods/a_test_register_file_auto.py
+++ b/accelerator/test_methods/a_test_register_file_auto.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test that the expected files are registered (and not registered) when
using job.open(), job.register_file() and job.register_files().
diff --git a/accelerator/test_methods/a_test_register_file_auto_2.py b/accelerator/test_methods/a_test_register_file_auto_2.py
index 9be661ea..1b420c04 100644
--- a/accelerator/test_methods/a_test_register_file_auto_2.py
+++ b/accelerator/test_methods/a_test_register_file_auto_2.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
import os
description = r'''
diff --git a/accelerator/test_methods/a_test_register_file_manual.py b/accelerator/test_methods/a_test_register_file_manual.py
index 3140b19b..04c83f94 100644
--- a/accelerator/test_methods/a_test_register_file_manual.py
+++ b/accelerator/test_methods/a_test_register_file_manual.py
@@ -17,11 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
-from accelerator.compat import PY2
import os
description = r'''
@@ -62,9 +57,4 @@ def synthesis(job):
os.mkdir('subdir/deep')
with open('subdir/deep/file.txt', 'w') as fh:
fh.write('written in a subdir, then registered with job.register_files() without a pattern.')
- if PY2:
- # No recursive support in glob in python 2.
- assert job.register_files() == set()
- assert job.register_files('*/*/*') == {'subdir/deep/file.txt'}
- else:
- assert job.register_files() == {'subdir/deep/file.txt'}
+ assert job.register_files() == {'subdir/deep/file.txt'}
diff --git a/accelerator/test_methods/a_test_selfchain.py b/accelerator/test_methods/a_test_selfchain.py
index d9a1cb70..21fc6cd5 100644
--- a/accelerator/test_methods/a_test_selfchain.py
+++ b/accelerator/test_methods/a_test_selfchain.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Tests creating several chained datasets in one job.
Exercises DatasetWriter.finish and the chaining logic
diff --git a/accelerator/test_methods/a_test_shell_commands.py b/accelerator/test_methods/a_test_shell_commands.py
index 1ff4dfa0..9730a7a4 100644
--- a/accelerator/test_methods/a_test_shell_commands.py
+++ b/accelerator/test_methods/a_test_shell_commands.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test all the ax sub-commands at least a little bit.
'''
@@ -55,12 +51,12 @@ def chk(cmd, want_in_help=[], want_in_call=[], dont_want_in_call=[]):
output = ax(args)
for want in want_l:
if want not in output:
- print("Expected to find %r in %r output:\n\n%s\n\nbut didn't." % (want, args, output,))
- raise Exception("Failed in command %r" % (cmd[0],))
+ print(f"Expected to find {want!r} in {args!r} output:\n\n{output}\n\nbut didn't.")
+ raise Exception(f"Failed in command {cmd[0]!r}")
for dont_want in dont_want_l:
if dont_want in output:
- print("Did not expect to find %r in %r output:\n\n%s\n\nbut did." % (dont_want, args, output,))
- raise Exception("Failed in command %r" % (cmd[0],))
+ print(f"Did not expect to find {dont_want!r} in {args!r} output:\n\n{output}\n\nbut did.")
+ raise Exception(f"Failed in command {cmd[0]!r}")
def synthesis(job):
print('look for this later')
diff --git a/accelerator/test_methods/a_test_shell_config.py b/accelerator/test_methods/a_test_shell_config.py
index 9a2525cb..7876bf9f 100644
--- a/accelerator/test_methods/a_test_shell_config.py
+++ b/accelerator/test_methods/a_test_shell_config.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test the alias and colour config for shell commands.
'''
@@ -62,10 +58,10 @@ def chk(cfgstr, pre, post):
try:
got = ax_cmd('job', job).split('\n')[-1]
except CalledProcessError as e:
- raise Exception("%r could not run: %s" % (cfgstr, e,))
- want = '%sWARNING: Job did not finish%s' % (pre, post,)
+ raise Exception(f"{cfgstr!r} could not run: {e}")
+ want = f'{pre}WARNING: Job did not finish{post}'
if got != want:
- raise Exception("%r:\nWanted %r\ngot %r" % (cfgstr, want, got,))
+ raise Exception(f"{cfgstr!r}:\nWanted {want!r}\ngot {got!r}")
# test the colour config
chk('\twarning = CYAN', '\x1b[36m', '\x1b[39m')
diff --git a/accelerator/test_methods/a_test_shell_data.py b/accelerator/test_methods/a_test_shell_data.py
index 8716b4a4..87c9c229 100644
--- a/accelerator/test_methods/a_test_shell_data.py
+++ b/accelerator/test_methods/a_test_shell_data.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Create some jobs and datasets to test the shell parser with.
diff --git a/accelerator/test_methods/a_test_shell_ds.py b/accelerator/test_methods/a_test_shell_ds.py
index c34df48b..16821da5 100644
--- a/accelerator/test_methods/a_test_shell_ds.py
+++ b/accelerator/test_methods/a_test_shell_ds.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test the "ax ds" shell command. Primarily tests the job spec parser.
'''
@@ -49,5 +45,4 @@ def synthesis(job):
res = ax_ds(spec)
got_ds = res[0]
ds = quote(ds)
- assert ds == got_ds, 'Spec %r should have given %r but gave %r' % (spec, ds, got_ds,)
-
+ assert ds == got_ds, f'Spec {spec!r} should have given {ds!r} but gave {got_ds!r}'
diff --git a/accelerator/test_methods/a_test_shell_grep.py b/accelerator/test_methods/a_test_shell_grep.py
index 00c1fe1d..963de00a 100644
--- a/accelerator/test_methods/a_test_shell_grep.py
+++ b/accelerator/test_methods/a_test_shell_grep.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test the "ax grep" shell command. This isn't testing complex regexes,
but rather the various output options and data types.
@@ -38,7 +34,7 @@
from itertools import cycle
from functools import partial
-from accelerator.compat import PY2, PY3, izip_longest
+from accelerator.compat import izip_longest
from accelerator.dsutil import _convfuncs
def grep_text(args, want, sep='\t', encoding='utf-8', unordered=False, check_output=check_output):
@@ -48,14 +44,14 @@ def grep_text(args, want, sep='\t', encoding='utf-8', unordered=False, check_out
res = check_output(cmd)
res = res.split(b'\n')[:-1]
if len(want) != len(res):
- raise Exception('%r gave %d lines, wanted %d.\n%r' % (cmd, len(res), len(want), res,))
+ raise Exception(f'{cmd!r} gave {len(res)} lines, wanted {len(want)}.\n{res!r}')
if encoding:
res = [el.decode(encoding, 'replace') for el in res]
typ = type(sep)
want = [sep.join(typ(el) for el in l) for l in want]
for lineno, (want, got) in enumerate(zip(want, res), 1):
if want != got:
- raise Exception('%r gave wrong result on line %d:\nWant: %r\nGot: %r' % (cmd, lineno, want, got,))
+ raise Exception(f'{cmd!r} gave wrong result on line {lineno}:\nWant: {want!r}\nGot: {got!r}')
# like subprocess.check_output except stdout is a pty
def check_output_pty(cmd):
@@ -85,21 +81,17 @@ def grep_json(args, want):
res = res.decode('utf-8', 'surrogatepass')
res = res.split('\n')[:-1]
if len(want) != len(res):
- raise Exception('%r gave %d lines, wanted %d.\n%r' % (cmd, len(res), len(want), res,))
+ raise Exception(f'{cmd!r} gave {len(res)} lines, wanted {len(want)}.\n{res!r}')
for lineno, (want, got) in enumerate(zip(want, res), 1):
try:
got = json.loads(got)
except Exception as e:
- raise Exception('%r made bad json %r on line %d: %s' % (cmd, got, lineno, e,))
+ raise Exception(f'{cmd!r} made bad json {got!r} on line {lineno}: {e}')
if want != got:
- raise Exception('%r gave wrong result on line %d:\nWant: %r\nGot: %r' % (cmd, lineno, want, got,))
+ raise Exception(f'{cmd!r} gave wrong result on line {lineno}:\nWant: {want!r}\nGot: {got!r}')
-if PY2:
- def mk_bytes(low, high):
- return b''.join(chr(c) for c in range(low, high))
-else:
- def mk_bytes(low, high):
- return bytes(range(low, high))
+def mk_bytes(low, high):
+ return bytes(range(low, high))
# looks like 'bar' when matching but 'foo' when printing
# intended to catch if objects are evaluated too many times
@@ -253,22 +245,21 @@ def frame(framevalues, *a):
], sep='', check_output=check_output_pty,
)
- if PY3: # no pickle type on PY2
- pickle = mk_ds('pickle', ['pickle'], [TricksyObject()], [''], [{'foo'}])
- grep_text(['', pickle], [['foo'], [''], ["{'foo'}"]])
- grep_text(['.', pickle], [['foo'], ["{'foo'}"]])
- grep_text(['bar', pickle], [['foo']])
- # using -g with the same columns as output is a NOP
- grep_text(['-g', 'pickle', 'bar', pickle], [['foo']])
- # but using it with a different set of columns is not
- pickle2 = mk_ds('pickle2', ['ascii'], ['a'], ['b'], ['c'], parent=pickle)
- grep_text(['-g', 'pickle', 'bar', pickle2], [['a', 'bar']])
- # order doesn't matter for equality, so here we're back to double evaluation.
- grep_text(['-g', 'pickle', '-g', 'ascii', 'bar', pickle2], [['a', 'foo']])
- bytespickle = mk_ds('bytespickle', ['pickle'], [b'\xf0'], [b'\t'])
- # pickles are str()d, not special cased like bytes columns
- grep_text(['-f', 'raw', 'xf0', bytespickle], [["b'\\xf0'"]])
- grep_json(['', bytespickle], [{'pickle': "b'\\xf0'"}, {'pickle': "b'\\t'"}])
+ pickle = mk_ds('pickle', ['pickle'], [TricksyObject()], [''], [{'foo'}])
+ grep_text(['', pickle], [['foo'], [''], ["{'foo'}"]])
+ grep_text(['.', pickle], [['foo'], ["{'foo'}"]])
+ grep_text(['bar', pickle], [['foo']])
+ # using -g with the same columns as output is a NOP
+ grep_text(['-g', 'pickle', 'bar', pickle], [['foo']])
+ # but using it with a different set of columns is not
+ pickle2 = mk_ds('pickle2', ['ascii'], ['a'], ['b'], ['c'], parent=pickle)
+ grep_text(['-g', 'pickle', 'bar', pickle2], [['a', 'bar']])
+ # order doesn't matter for equality, so here we're back to double evaluation.
+ grep_text(['-g', 'pickle', '-g', 'ascii', 'bar', pickle2], [['a', 'foo']])
+ bytespickle = mk_ds('bytespickle', ['pickle'], [b'\xf0'], [b'\t'])
+ # pickles are str()d, not special cased like bytes columns
+ grep_text(['-f', 'raw', 'xf0', bytespickle], [["b'\\xf0'"]])
+ grep_json(['', bytespickle], [{'pickle': "b'\\xf0'"}, {'pickle': "b'\\t'"}])
# --only-matching, both the part (default) and columns (with -l) in both csv and json
grep_text(['-o', '-c', '1', b], [['1', ''], ['11', '1'], ['1'], ['11']])
@@ -282,11 +273,8 @@ def frame(framevalues, *a):
['printable', mk_bytes(32, 128)],
['not ascii', mk_bytes(128, 256)],
)
- if PY2:
- encoded_not_ascii = raw_not_ascii = '\ufffd'.encode('utf-8') * 128
- else:
- raw_not_ascii = mk_bytes(128, 256)
- encoded_not_ascii = raw_not_ascii.decode('utf-8', 'surrogateescape').encode('utf-8', 'surrogatepass')
+ raw_not_ascii = mk_bytes(128, 256)
+ encoded_not_ascii = raw_not_ascii.decode('utf-8', 'surrogateescape').encode('utf-8', 'surrogatepass')
grep_text(
['--format=raw', '', allbytes],
[
@@ -309,9 +297,9 @@ def frame(framevalues, *a):
sep=b'\t',
)
grep_json(['', allbytes], [
- {'ascii': 'control chars', 'bytes': mk_bytes(0, 32).decode('utf-8', 'surrogateescape' if PY3 else 'replace')},
- {'ascii': 'printable', 'bytes': mk_bytes(32, 128).decode('utf-8', 'surrogateescape' if PY3 else 'replace')},
- {'ascii': 'not ascii', 'bytes': mk_bytes(128, 256).decode('utf-8', 'surrogateescape' if PY3 else 'replace')},
+ {'ascii': 'control chars', 'bytes': mk_bytes(0, 32).decode('utf-8', 'surrogateescape')},
+ {'ascii': 'printable', 'bytes': mk_bytes(32, 128).decode('utf-8', 'surrogateescape')},
+ {'ascii': 'not ascii', 'bytes': mk_bytes(128, 256).decode('utf-8', 'surrogateescape')},
])
# header printing should happen between datasets only when columns change,
@@ -332,7 +320,7 @@ def frame(framevalues, *a):
slice = 0
header_test = []
for ds_ix, cols in enumerate(columns):
- dw = job.datasetwriter(name='header test %d' % (ds_ix,), previous=previous, allow_missing_slices=True)
+ dw = job.datasetwriter(name=f'header test {ds_ix}', previous=previous, allow_missing_slices=True)
for col in cols:
dw.add(col, col)
if sorted(cols) != previous_cols:
@@ -436,7 +424,7 @@ def frame(framevalues, *a):
# test escaping
unescaped, sliceno = header_test[-1]
escaped = unescaped.replace('\n', '\\n').replace('\r', '\\r').replace('"', '""')
- grep_text(['-l', '-t', '/', '-S', '', previous], [['"%s"/%d' % (escaped, sliceno)]])
+ grep_text(['-l', '-t', '/', '-S', '', previous], [[f'"{escaped}"/{sliceno}']])
# more escaping
escapy = mk_ds('escapy',
@@ -620,11 +608,9 @@ def frame(framevalues, *a):
grep_text(['-g', 'json', 'foo', alltypes], [])
grep_text(['-g', 'bytes', 'tet', alltypes, 'ascii', 'unicode'], [['foo', 'codepoints\x00\xe4']])
grep_text(['-g', 'bytes', '\\x00', alltypes, 'bool'], [['True']])
- if PY3:
- # python2 doesn't really handle non-utf8 bytes
- grep_text(['-g', 'bytes', '\\udcff', alltypes, 'bool'], [['True']])
- grep_text(['--format=raw', '-g', 'json', '-i', 'foo', alltypes], [[b'foo', b'True', b'\xff\x00octets' if PY3 else b'\xef\xbf\xbd\x00octets', b'(1+2j)', b'(1.5-0.5j)', b'2021-09-20', b'2021-09-20 01:02:03', b'0.125', b'1e+42', b"[1, 2, 3, {'FOO': 'BAR'}, None]" if PY3 else b"[1, 2, 3, {u'FOO': u'BAR'}, None]", b'-2', b'04:05:06', b'codepoints\x00\xc3\xa4']], sep=b'\t', encoding=None)
- grep_json([':05:', alltypes, 'bool', 'time', 'unicode', 'bytes'], [{'bool': True, 'time': '04:05:06', 'unicode': 'codepoints\x00\xe4', 'bytes': '\udcff\x00octets' if PY3 else '\ufffd\x00octets'}])
+ grep_text(['-g', 'bytes', '\\udcff', alltypes, 'bool'], [['True']])
+ grep_text(['--format=raw', '-g', 'json', '-i', 'foo', alltypes], [[b'foo', b'True', b'\xff\x00octets', b'(1+2j)', b'(1.5-0.5j)', b'2021-09-20', b'2021-09-20 01:02:03', b'0.125', b'1e+42', b"[1, 2, 3, {'FOO': 'BAR'}, None]", b'-2', b'04:05:06', b'codepoints\x00\xc3\xa4']], sep=b'\t', encoding=None)
+ grep_json([':05:', alltypes, 'bool', 'time', 'unicode', 'bytes'], [{'bool': True, 'time': '04:05:06', 'unicode': 'codepoints\x00\xe4', 'bytes': '\udcff\x00octets'}])
columns = [
'ascii',
@@ -695,9 +681,7 @@ def json_fixup(line):
{'dataset': d, 'sliceno': 1, 'lineno': 0, 'data': want_json[1]},
])
all_types = {n for n in _convfuncs if not n.startswith('parsed:')}
- if PY2:
- all_types.remove('pickle')
- assert used_types == all_types, 'Missing/extra column types: %r %r' % (all_types - used_types, used_types - all_types,)
+ assert used_types == all_types, f'Missing/extra column types: {all_types - used_types!r} {used_types - all_types!r}'
# test the smart tab mode with various lengths
@@ -1165,7 +1149,7 @@ def mk_lined_ds(name, *lines, **kw):
cmd = options.command_prefix + ['grep', '--unique=c', '--chain', '000', uniq3]
try:
check_output(cmd)
- raise Exception("%r worked, should have complained that %s doesn't have column c" % (cmd, uniq.quoted,))
+ raise Exception(f"{cmd!r} worked, should have complained that {uniq.quoted} doesn't have column c")
except CalledProcessError:
pass
diff --git a/accelerator/test_methods/a_test_shell_job.py b/accelerator/test_methods/a_test_shell_job.py
index b6ede9f9..1ce5e4f0 100644
--- a/accelerator/test_methods/a_test_shell_job.py
+++ b/accelerator/test_methods/a_test_shell_job.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test the "ax job" shell command. Primarily tests the job spec parser.
'''
@@ -56,4 +52,4 @@ def synthesis(job):
for spec, jobid in options.want.items():
res = ax_job(spec)
got_jobid = res[0].split('/')[-1]
- assert got_jobid.startswith(jobid), 'Spec %r should have given %r but gave %r' % (spec, jobid, got_jobid,)
+ assert got_jobid.startswith(jobid), f'Spec {spec!r} should have given {jobid!r} but gave {got_jobid!r}'
diff --git a/accelerator/test_methods/a_test_sort_chaining.py b/accelerator/test_methods/a_test_sort_chaining.py
index 73ef00c1..83f59eea 100644
--- a/accelerator/test_methods/a_test_sort_chaining.py
+++ b/accelerator/test_methods/a_test_sort_chaining.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test dataset_sort as a chain, across a chain and as a chain merging
only two datasets of the original chain.
diff --git a/accelerator/test_methods/a_test_sort_stability.py b/accelerator/test_methods/a_test_sort_stability.py
index 9d79e29b..85c9c5ab 100644
--- a/accelerator/test_methods/a_test_sort_stability.py
+++ b/accelerator/test_methods/a_test_sort_stability.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test that dataset_sort sorts stably.
diff --git a/accelerator/test_methods/a_test_sort_trigger.py b/accelerator/test_methods/a_test_sort_trigger.py
index be24f72c..60869ff3 100644
--- a/accelerator/test_methods/a_test_sort_trigger.py
+++ b/accelerator/test_methods/a_test_sort_trigger.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test dataset_sort with trigger_column
'''
@@ -32,7 +28,7 @@
def sort(src, ix, **kw):
ds = subjobs.build('dataset_sort', source=src, sort_across_slices=True, **kw).dataset()
want = sorted(src.iterate(None), key=itemgetter(ix))
- assert list(ds.iterate(None)) == want, '%s != sorted(%s)' % (ds, src,)
+ assert list(ds.iterate(None)) == want, f'{ds} != sorted({src})'
return ds
def synthesis(job):
@@ -51,14 +47,14 @@ def synthesis(job):
src = dw.finish()
# Unchanging trigger
a = sort(src, 0, sort_columns='a', trigger_column='a')
- assert set(a.lines) == {0, 60}, '%s %r' % (a, a.lines,)
+ assert set(a.lines) == {0, 60}, f'{a} {a.lines!r}'
# Just two trigger values
b = sort(src, 1, sort_columns='b', trigger_column='b')
- assert set(b.lines) == {0, 30}, '%s %r' % (b, b.lines,)
+ assert set(b.lines) == {0, 30}, f'{b} {b.lines!r}'
# Trigger value changes every time - trigger_column should affect nothing
c = sort(src, 2, sort_columns='c')
ct = sort(src, 2, sort_columns='c', trigger_column='c')
- assert c.lines == ct.lines, '%s %r != %s %r' % (c, c.lines, ct, ct.lines,)
+ assert c.lines == ct.lines, f'{c} {c.lines!r} != {ct} {ct.lines!r}'
# check that using second sort column as trigger works
bc = sort(src, 2, sort_columns=['c', 'b'], trigger_column='b')
- assert set(bc.lines) == {0, 30}, '%s %r' % (bc, bc.lines,)
+ assert set(bc.lines) == {0, 30}, f'{bc} {bc.lines!r}'
diff --git a/accelerator/test_methods/a_test_sorting.py b/accelerator/test_methods/a_test_sorting.py
index c50eafd3..fceb7e24 100644
--- a/accelerator/test_methods/a_test_sorting.py
+++ b/accelerator/test_methods/a_test_sorting.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test dataset_sort with various options on a dataset with all types.
'''
@@ -87,7 +83,7 @@ def cmp(a, b):
for sliceno in range(slices):
good = sorted(test_data.sort_data_for_slice(sliceno), key=keycmp, reverse=reverse)
check = list(ds.iterate(sliceno))
- assert unnan(check) == unnan(good), "Slice %d sorted on %s bad (%s)" % (sliceno, key, jid,)
+ assert unnan(check) == unnan(good), f"Slice {sliceno} sorted on {key} bad ({jid})"
def synthesis(params):
source = Dataset(subjobs.build("test_sorting_gendata"))
@@ -113,4 +109,4 @@ def synthesis(params):
good = sorted(all_data, key=lambda t: (noneninf(t[int64_off]), noneninf(t[int32_off]),), reverse=True)
ds = Dataset(jid)
check = list(ds.iterate(None))
- assert unnan(check) == unnan(good), "Sorting across slices on [int64, int32] bad (%s)" % (jid,)
+ assert unnan(check) == unnan(good), f"Sorting across slices on [int64, int32] bad ({jid})"
diff --git a/accelerator/test_methods/a_test_sorting_gendata.py b/accelerator/test_methods/a_test_sorting_gendata.py
index cfb25d10..ea8d5b5e 100644
--- a/accelerator/test_methods/a_test_sorting_gendata.py
+++ b/accelerator/test_methods/a_test_sorting_gendata.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Generate data for test_sorting.
'''
diff --git a/accelerator/test_methods/a_test_status_in_exceptions.py b/accelerator/test_methods/a_test_status_in_exceptions.py
index 946f2630..84a647b5 100644
--- a/accelerator/test_methods/a_test_status_in_exceptions.py
+++ b/accelerator/test_methods/a_test_status_in_exceptions.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test that the JobError exception contains the correct status stack
and line number from dataset iteration.
@@ -114,7 +110,7 @@ def synthesis(job):
want_re += '.*' + status1
for ix, want_line in enumerate(on_line, 1):
ds = job.dataset(str(len(on_line) - ix))
- want_re += '.*Iterating %s:0 reached line %d\n' % (re.escape(ds.quoted), want_line,)
+ want_re += f'.*Iterating {re.escape(ds.quoted)}:0 reached line {want_line}\n'
if status2:
want_re += '.*' + status2
assert re.search(want_re, got, re.DOTALL), got
diff --git a/accelerator/test_methods/a_test_subjobs_nesting.py b/accelerator/test_methods/a_test_subjobs_nesting.py
index 7be08f91..c746d378 100644
--- a/accelerator/test_methods/a_test_subjobs_nesting.py
+++ b/accelerator/test_methods/a_test_subjobs_nesting.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Verify that subjobs are allowed to nest (exactly) five levels.
'''
diff --git a/accelerator/test_methods/a_test_subjobs_type.py b/accelerator/test_methods/a_test_subjobs_type.py
index 2c8b4d4f..b7810671 100644
--- a/accelerator/test_methods/a_test_subjobs_type.py
+++ b/accelerator/test_methods/a_test_subjobs_type.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Type datasets.untyped the same as datasets.typed in a subjob, then verify
(in another subjob) the the results are correct.
diff --git a/accelerator/test_methods/a_test_summary.py b/accelerator/test_methods/a_test_summary.py
index 089f6d6e..0b0b6a2c 100644
--- a/accelerator/test_methods/a_test_summary.py
+++ b/accelerator/test_methods/a_test_summary.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Produce a summary that build_tests can put in the result directory.
'''
diff --git a/accelerator/test_methods/a_test_urd.py b/accelerator/test_methods/a_test_urd.py
index ae305d1b..34f28e63 100644
--- a/accelerator/test_methods/a_test_urd.py
+++ b/accelerator/test_methods/a_test_urd.py
@@ -17,10 +17,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
description = r'''
Test a stand-alone urd for compatibility with old logs and some calls.
'''
@@ -29,7 +25,7 @@
command_prefix=['ax', '--config', '/some/path/here'],
)
-from accelerator.compat import open, url_quote_more
+from accelerator.compat import url_quote_more
from accelerator.error import UrdPermissionError, UrdConflictError
from accelerator.unixhttp import call
from subprocess import Popen
@@ -71,7 +67,7 @@ def synthesis(job):
headers = {'Content-Type': 'application/json', 'Authorization': 'Basic dGVzdDpwYXNz'}
def check(url_part, want, post_data=None):
got = call(url + url_part, server_name='urd', data=post_data, headers=headers, fmt=json.loads)
- assert want == got, '\nWanted %r,\ngot %r' % (want, got,)
+ assert want == got, f'\nWanted {want!r},\ngot {got!r}'
check('list', ['test/ing', 'test/two'])
check('test/ing/since/0', ['2023-01', '2023-02', '2023-06', '2024-03'])
@@ -130,7 +126,7 @@ def check(url_part, want, post_data=None):
for got, want in zip(fh, want_it):
assert got.startswith('4\t'), got
got = got.split('\t', 2)[2]
- assert want == got, '\nWanted %r,\ngot %r' % (want, got,)
+ assert want == got, f'\nWanted {want!r},\ngot {got!r}'
assert next(want_it) == 'END'
p.terminate()
diff --git a/accelerator/test_methods/build_tests.py b/accelerator/test_methods/build_tests.py
index dc3f5e51..27211aca 100644
--- a/accelerator/test_methods/build_tests.py
+++ b/accelerator/test_methods/build_tests.py
@@ -18,10 +18,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
from accelerator.dataset import Dataset
from accelerator.build import JobError
from accelerator.compat import monotonic
@@ -37,7 +33,7 @@
'''
def main(urd):
- assert urd.info.slices >= 3, "The tests don't work with less than 3 slices (you have %d)." % (urd.info.slices,)
+ assert urd.info.slices >= 3, f"The tests don't work with less than 3 slices (you have {urd.info.slices})."
from accelerator.shell import cfg
# sys.argv[0] is absolute for unqualified commands ("ax"), but exactly what
@@ -84,14 +80,14 @@ def main(urd):
assert fin == {'new': False, 'changed': False, 'is_ghost': False}, fin
urd.begin("tests.urd", 1) # will be overridden to 2 in finish
jl = urd.latest("tests.urd").joblist
- assert jl == [job], '%r != [%r]' % (jl, job,)
+ assert jl == [job], f'{jl!r} != [{job!r}]'
urd.build("test_build_kws", options=dict(foo='bar', a='A'))
urd.finish("tests.urd", 2, caption="second")
u = urd.peek_latest("tests.urd")
assert u.caption == "second"
dep0 = list(u.deps.values())[0]
assert dep0.caption == "first", dep0.caption
- assert dep0.joblist == jl, '%r != %r' % (dep0.joblist, jl,)
+ assert dep0.joblist == jl, f'{dep0.joblist!r} != {jl!r}'
assert urd.since("tests.urd", 0) == ['1', '2']
urd.truncate("tests.urd", 2)
assert urd.since("tests.urd", 0) == ['1']
@@ -129,22 +125,22 @@ def main(urd):
for how in ("exiting", "dying",):
print()
- print("Verifying that an analysis process %s kills the job" % (how,))
+ print(f"Verifying that an analysis process {how} kills the job")
time_before = monotonic()
try:
job = urd.build("test_analysis_died", how=how)
- print("test_analysis_died completed successfully (%s), that shouldn't happen" % (job,))
+ print(f"test_analysis_died completed successfully ({job}), that shouldn't happen")
exit(1)
except JobError:
time_after = monotonic()
time_to_die = time_after - time_before
if time_to_die > 13:
- print("test_analysis_died took %d seconds to die, it should be faster" % (time_to_die,))
+ print(f"test_analysis_died took {time_to_die} seconds to die, it should be faster")
exit(1)
elif time_to_die > 2:
- print("test_analysis_died took %d seconds to die, so death detection is slow, but works" % (time_to_die,))
+ print(f"test_analysis_died took {time_to_die} seconds to die, so death detection is slow, but works")
else:
- print("test_analysis_died took %.1f seconds to die, so death detection works" % (time_to_die,))
+ print(f"test_analysis_died took {time_to_die:.1f} seconds to die, so death detection works")
print()
print("Testing dataset creation, export, import")
diff --git a/accelerator/test_methods/test_data.py b/accelerator/test_methods/test_data.py
index a0bc5870..752c2dc9 100644
--- a/accelerator/test_methods/test_data.py
+++ b/accelerator/test_methods/test_data.py
@@ -19,10 +19,6 @@
# Test data for use in dataset testing
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-
from datetime import date, time, datetime
from sys import version_info
diff --git a/accelerator/unixhttp.py b/accelerator/unixhttp.py
index 7cf5a6a7..672aa613 100644
--- a/accelerator/unixhttp.py
+++ b/accelerator/unixhttp.py
@@ -19,21 +19,13 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-
-from accelerator.compat import PY3
from accelerator.compat import urlopen, Request, URLError, HTTPError
from accelerator.extras import json_encode, json_decode
from accelerator.error import ServerError, UrdError, UrdPermissionError, UrdConflictError
from accelerator import g, __version__ as ax_version
-if PY3:
- from urllib.request import install_opener, build_opener, AbstractHTTPHandler
- from http.client import HTTPConnection
-else:
- from urllib2 import install_opener, build_opener, AbstractHTTPHandler
- from httplib import HTTPConnection
+from urllib.request import install_opener, build_opener, AbstractHTTPHandler
+from http.client import HTTPConnection
import sys
import time
@@ -92,10 +84,9 @@ def call(url, data=None, fmt=json_decode, headers={}, server_name='server', retr
s_version = r.headers['Accelerator-Version'] or ''
if s_version != ax_version:
# Nothing is supposed to catch this, so just print and die.
- print('Server is running version %s but we are running version %s' % (s_version, ax_version,), file=sys.stderr)
+ print(f'Server is running version {s_version} but we are running version {ax_version}', file=sys.stderr)
exit(1)
- if PY3:
- resp = resp.decode('utf-8')
+ resp = resp.decode('utf-8')
# It is inconsistent if we get HTTPError or not.
# It seems we do when using TCP sockets, but not when using unix sockets.
if r.getcode() >= 400:
@@ -108,10 +99,8 @@ def call(url, data=None, fmt=json_decode, headers={}, server_name='server', retr
pass
except HTTPError as e:
if resp is None and e.fp:
- resp = e.fp.read()
- if PY3:
- resp = resp.decode('utf-8')
- msg = '%s says %d: %s' % (server_name, e.code, resp,)
+ resp = e.fp.read().decode('utf-8')
+ msg = f'{server_name} says {e.code}: {resp}'
if server_name == 'urd' and 400 <= e.code < 500:
if e.code == 401:
err = UrdPermissionError()
@@ -126,15 +115,15 @@ def call(url, data=None, fmt=json_decode, headers={}, server_name='server', retr
if attempt < retries - 1:
msg = None
else:
- msg = 'error contacting %s: %s' % (server_name, e.reason)
+ msg = f'error contacting {server_name}: {e.reason}'
except ValueError as e:
- msg = 'Bad data from %s, %s: %s' % (server_name, type(e).__name__, e,)
+ msg = f'Bad data from {server_name}, {type(e).__name__}: {e}'
if msg and not quiet:
print(msg, file=sys.stderr)
if attempt < retries + 1:
time.sleep(attempt / 15)
if msg and not quiet:
- print('Retrying (%d/%d).' % (attempt, retries,), file=sys.stderr)
+ print(f'Retrying ({attempt}/{retries}).', file=sys.stderr)
else:
if not quiet:
print('Giving up.', file=sys.stderr)
diff --git a/accelerator/urd.py b/accelerator/urd.py
index f57e36b7..fa0ef03a 100644
--- a/accelerator/urd.py
+++ b/accelerator/urd.py
@@ -19,10 +19,6 @@
# #
############################################################################
-from __future__ import unicode_literals
-from __future__ import print_function
-from __future__ import division
-
from glob import glob
from collections import defaultdict
from bottle import route, request, auth_basic, abort
@@ -38,8 +34,7 @@
import os
import signal
-from accelerator.compat import iteritems, itervalues, unicode
-from accelerator.compat import PY3
+from accelerator.compat import iteritems, itervalues
from accelerator.colourwrapper import colour
from accelerator.shell.parser import ArgumentParser
from accelerator.unixhttp import WaitressServer
@@ -61,7 +56,7 @@ def joblistlike(jl):
assert isinstance(v, (list, tuple)), v
assert len(v) == 2, v
for s in v:
- assert isinstance(s, unicode), s
+ assert isinstance(s, str), s
return True
@@ -73,26 +68,25 @@ class TimeStamp(str):
Extra parts sort after.
"""
- if PY3: # python2 doesn't support slots on str subclasses
- __slots__ = ('_parts')
+ __slots__ = ('_parts')
def __new__(cls, ts):
if isinstance(ts, TimeStamp):
return ts
- assert ts, 'Invalid timestamp %s' % (ts,)
+ assert ts, f'Invalid timestamp {ts}'
parts = []
str_parts = []
for part in ts.split('+'):
try:
integer = int(part, 10)
- assert integer >= 0, 'Invalid timestamp %d' % (part,)
+ assert integer >= 0, f'Invalid timestamp {part}'
parts.append((0, integer,))
str_parts.append(str(integer))
continue
except ValueError:
pass
m = re.match(r'(\d{4}-\d{2}(?:-\d{2}(?:[T ]\d{2}(?::\d{2}(?::\d{2}(?:\.\d{1,6})?)?)?)?)?)$', part)
- assert m, 'Invalid timestamp %s' % (part,)
+ assert m, f'Invalid timestamp {part}'
part = part.replace(' ', 'T')
parts.append((1, part,))
str_parts.append(part)
@@ -178,10 +172,10 @@ def __init__(self, path, verbose=True):
if verbose:
print("urd-list lines ghosts active")
for key, val in sorted(stat.items()):
- print("%-30s %7d %7d %7d" % (key, val, len(self.ghost_db[key]), len(self.db[key]),))
+ print(f"{key:30} {val:7} {len(self.ghost_db[key]):7} {len(self.db[key]):7}")
print()
else:
- print("Creating directory \"%s\"." % (path,))
+ print(f"Creating directory \"{path}\".")
os.makedirs(path)
self._lasttime = None
self._initialised = True
@@ -236,9 +230,9 @@ def _parse_truncate(self, line):
def _validate_data(self, data, with_deps=True):
if with_deps:
assert set(data) == {'timestamp', 'joblist', 'caption', 'user', 'build', 'deps', 'flags', 'build_job',}
- assert isinstance(data.user, unicode)
- assert isinstance(data.build, unicode)
- assert isinstance(data.build_job, unicode)
+ assert isinstance(data.user, str)
+ assert isinstance(data.build, str)
+ assert isinstance(data.build_job, str)
assert isinstance(data.deps, dict)
for v in itervalues(data.deps):
assert isinstance(v, dict)
@@ -247,7 +241,7 @@ def _validate_data(self, data, with_deps=True):
assert set(data) == {'timestamp', 'joblist', 'caption',}
assert joblistlike(data.joblist), data.joblist
assert data.joblist
- assert isinstance(data.caption, unicode)
+ assert isinstance(data.caption, str)
data.timestamp = TimeStamp(data.timestamp)
def _serialise(self, action, data):
@@ -257,7 +251,7 @@ def _serialise(self, action, data):
json_joblist = json.dumps(data.joblist)
json_caption = json.dumps(data.caption)
json_build_job = json.dumps(data.build_job)
- key = '%s/%s' % (data.user, data.build,)
+ key = f'{data.user}/{data.build}'
flags = ','.join(data.flags)
for s in key, json_deps, json_joblist, json_caption, data.user, data.build, data.timestamp, flags:
assert '\t' not in s, s
@@ -289,9 +283,9 @@ def _is_ghost(self, data):
@locked
def add(self, data):
- key = '%s/%s' % (data.user, data.build)
+ key = f'{data.user}/{data.build}'
flags = data.pop('flags', [])
- assert flags in ([], ['update']), 'Unknown flags: %r' % (flags,)
+ assert flags in ([], ['update']), f'Unknown flags: {flags!r}'
new = False
changed = False
ghosted = 0
@@ -313,7 +307,7 @@ def add(self, data):
else:
new = True
if changed and 'update' not in flags:
- assert self._initialised, 'Log updates without update flag: %r' % (data,)
+ assert self._initialised, f'Log updates without update flag: {data!r}'
bottle.response.status = 409
return {'error': 'would update'}
if new or changed:
@@ -402,7 +396,7 @@ def log(self, action, data):
extra = " Also failed to remove partially written data."
stars = colour.red('****')
extra2 = " " + stars + " YOUR URD DB IS PROBABLY BROKEN NOW! " + stars
- msg = " Failed to write %s: %s" % (fn, e)
+ msg = f" Failed to write {fn}: {e}"
brk = "#" * (max(len(msg), len(extra)) + 2)
print("", file=sys.stderr)
print(brk, file=sys.stderr)
@@ -512,9 +506,7 @@ def cmpfunc(k, ts):
@route('/add', method='POST')
@auth_basic(auth)
def add():
- body = request.body
- if PY3:
- body = TextIOWrapper(body, encoding='utf-8')
+ body = TextIOWrapper(request.body, encoding='utf-8')
data = Entry(json.load(body))
if data.user != request.auth[0]:
abort(401, "Error: user does not match authentication!")
@@ -608,7 +600,7 @@ def main(argv, cfg):
authdict = readauth(auth_fn)
allow_passwordless = args.allow_passwordless
if not authdict and not args.allow_passwordless:
- raise Exception('No users in %r and --allow-passwordless not specified.' % (auth_fn,))
+ raise Exception(f'No users in {auth_fn!r} and --allow-passwordless not specified.')
db = DB(args.path, not args.quiet)
bottle.install(jsonify)
diff --git a/accelerator/web.py b/accelerator/web.py
index d3871ff4..6a228484 100644
--- a/accelerator/web.py
+++ b/accelerator/web.py
@@ -18,19 +18,10 @@
# #
############################################################################
-from accelerator.compat import PY3, unicode
-
-if PY3:
- from socketserver import ThreadingMixIn
- from http.server import HTTPServer, BaseHTTPRequestHandler
- from socketserver import UnixStreamServer
- from urllib.parse import parse_qs, unquote_plus
-else:
- from SocketServer import ThreadingMixIn
- from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
- from SocketServer import UnixStreamServer
- from urlparse import parse_qs
- from urllib import unquote_plus
+from socketserver import ThreadingMixIn
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from socketserver import UnixStreamServer
+from urllib.parse import parse_qs, unquote_plus
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
request_queue_size = 512
@@ -76,8 +67,7 @@ def do_POST(self):
bare_ctype = ctype.split(";", 1)[0].strip()
if bare_ctype == "application/x-www-form-urlencoded":
cgi_args = parse_qs(data, keep_blank_values=True)
- if PY3:
- cgi_args = {k.decode('ascii'): v for k, v in cgi_args.items()}
+ cgi_args = {k.decode('ascii'): v for k, v in cgi_args.items()}
else:
cgi_args = {None: [data]}
self.is_head = False
@@ -97,7 +87,7 @@ def _bad_request(self):
def argdec(self, v):
if self.unicode_args:
- if type(v) is unicode: return v
+ if type(v) is str: return v
try:
return v.decode("utf-8")
except Exception:
diff --git a/accelerator/workspace.py b/accelerator/workspace.py
index 195d5715..860d6f18 100644
--- a/accelerator/workspace.py
+++ b/accelerator/workspace.py
@@ -19,9 +19,6 @@
# #
############################################################################
-from __future__ import print_function
-from __future__ import division
-
import os
from accelerator.job import Job
@@ -48,7 +45,7 @@ def _check_metafile(self):
""" verify or write metadata file in workdir """
filename = os.path.join(self.path, ".slices")
if not os.path.isdir(self.path):
- print('\nERROR: Directory \"%s\" does not exist!' % (self.path,))
+ print(f'\nERROR: Directory "{self.path}" does not exist!')
return False
if not os.path.exists(filename):
print('\nCreate ' + filename)
@@ -61,7 +58,7 @@ def _check_metafile(self):
with open(filename) as F:
file_slices = int(F.read())
if file_slices != self.slices:
- print('\nERROR: Number of slices in workdir \"%s\" differs from config file!' % (self.name,))
+ print(f'\nERROR: Number of slices in workdir "{self.name}" differs from config file!')
return False
return True
@@ -103,7 +100,7 @@ def allocate_jobs(self, num_jobs):
jobidv = [Job._create(self.name, highest + 1 + x) for x in range(num_jobs)]
for jobid in jobidv:
fullpath = os.path.join(self.path, jobid)
- print("WORKDIR: Allocate_job \"%s\"" % fullpath)
+ print(f"WORKDIR: Allocate_job \"{fullpath}\"")
self.known_jobids.add(jobid)
os.mkdir(fullpath)
return jobidv
diff --git a/dsutil/test.py b/dsutil/test.py
index e180526d..0ef97063 100755
--- a/dsutil/test.py
+++ b/dsutil/test.py
@@ -22,8 +22,6 @@
# Verify general operation and a few corner cases.
-from __future__ import division, print_function, unicode_literals
-
from datetime import datetime, date, time
from sys import version_info
from itertools import compress
@@ -92,7 +90,7 @@ def can_minmax(name):
if typ is w_typ:
# File is not created until written.
fh.flush()
- raise Exception("%r does not give IOError for DOES/NOT/EXIST" % (typ,))
+ raise Exception(f"{typ!r} does not give IOError for DOES/NOT/EXIST")
except IOError:
pass
try:
@@ -101,7 +99,7 @@ def can_minmax(name):
except TypeError:
pass
except Exception:
- raise Exception("%r does not give TypeError for bad keyword argument" % (typ,))
+ raise Exception(f"{typ!r} does not give TypeError for bad keyword argument")
# test that the right data fails to write
for test_none_support in (False, True):
with w_mk(TMP_FN, none_support=test_none_support) as fh:
@@ -116,12 +114,12 @@ def can_minmax(name):
raise Exception("Allowed None without none_support")
except (ValueError, TypeError, OverflowError):
assert ix < bad_cnt or (value is None and not test_none_support), repr(value)
- assert fh.count == count, "%s: %d lines written, claims %d" % (name, count, fh.count,)
+ assert fh.count == count, f"{name}: {count} lines written, claims {fh.count}"
if can_minmax(name):
want_min = min(filter(lambda x: x is not None, res_data))
want_max = max(filter(lambda x: x is not None, res_data))
- assert fh.min == want_min, "%s: claims min %r, not %r" % (name, fh.min, want_min,)
- assert fh.max == want_max, "%s: claims max %r, not %r" % (name, fh.max, want_max,)
+ assert fh.min == want_min, f"{name}: claims min {fh.min!r}, not {want_min!r}"
+ assert fh.max == want_max, f"{name}: claims max {fh.max!r}, not {want_max!r}"
# Okay, errors look good
with r_mk(TMP_FN) as fh:
res = list(fh)
@@ -151,8 +149,8 @@ def can_minmax(name):
fh.write(value)
count += 1
except (ValueError, TypeError, OverflowError):
- assert 0, "No default: %r" % (value,)
- assert fh.count == count, "%s: %d lines written, claims %d" % (name, count, fh.count,)
+ assert 0, f"No default: {value!r}"
+ assert fh.count == count, f"{name}: {count} lines written, claims {fh.count}"
# No errors when there is a default
with r_mk(TMP_FN) as fh:
res = list(fh)
@@ -175,17 +173,17 @@ def slice_test(slices, spread_None):
assert hc == wrote, "Hashcheck disagrees with write"
except (ValueError, TypeError, OverflowError):
assert ix < bad_cnt, repr(value)
- assert fh.count == count, "%s (%d, %d): %d lines written, claims %d" % (name, sliceno, slices, count, fh.count,)
+ assert fh.count == count, f"{name} ({sliceno}, {slices}): {count} lines written, claims {fh.count}"
if not forstrings(name):
got_min, got_max = fh.min, fh.max
fh.flush() # we overwrite the same file, so make sure we write.
total_count += count
with r_mk(TMP_FN) as fh:
tmp = list(fh)
- assert len(tmp) == count, "%s (%d, %d): %d lines written, claims %d" % (name, sliceno, slices, len(tmp), count,)
+ assert len(tmp) == count, f"{name} ({sliceno}, {slices}): {len(tmp)} lines written, claims {count}"
for v in tmp:
- assert (spread_None and v is None) or w_typ.hash(v) % slices == sliceno, "Bad hash for %r" % (v,)
- assert w_typ.hash(v) == _dsutil.hash(v), "Inconsistent hash for %r" % (v,)
+ assert (spread_None and v is None) or w_typ.hash(v) % slices == sliceno, f"Bad hash for {v!r}"
+ assert w_typ.hash(v) == _dsutil.hash(v), f"Inconsistent hash for {v!r}"
res.extend(tmp)
sliced_res.append(tmp)
if can_minmax(name):
@@ -193,13 +191,13 @@ def slice_test(slices, spread_None):
if tmp:
want_min = min(tmp)
want_max = max(tmp)
- assert got_min == want_min, "%s (%d, %d): claims min %r, not %r" % (name, sliceno, slices, got_min, want_min,)
- assert got_max == want_max, "%s (%d, %d): claims max %r, not %r" % (name, sliceno, slices, got_max, want_max,)
+ assert got_min == want_min, f"{name} ({sliceno}, {slices}): claims min {got_min!r}, not {want_min!r}"
+ assert got_max == want_max, f"{name} ({sliceno}, {slices}): claims max {got_max!r}, not {want_max!r}"
else:
assert got_min is None and got_max is None
- assert len(res) == total_count, "%s (%d): %d lines written, claims %d" % (name, slices, len(res), total_count,)
- assert len(res) == len(res_data), "%s (%d): %d lines written, should be %d" % (name, slices, len(res), len(res_data),)
- assert set(res) == set(res_data), "%s (%d): Wrong data: %r != %r" % (name, slices, res, res_data,)
+ assert len(res) == total_count, f"{name} ({slices}): {len(res)} lines written, claims {total_count}"
+ assert len(res) == len(res_data), f"{name} ({slices}): {len(res)} lines written, should be {len(res_data)}"
+ assert set(res) == set(res_data), f"{name} ({slices}): Wrong data: {res!r} != {res_data!r}"
# verify reading back with hashfilter gives the same as writing with it
with w_mk(TMP_FN, none_support=True) as fh:
for value in data[bad_cnt:]:
@@ -207,7 +205,7 @@ def slice_test(slices, spread_None):
for sliceno in range(slices):
with r_mk(TMP_FN, hashfilter=(sliceno, slices, spread_None)) as fh:
slice_values = list(compress(res_data, fh))
- assert slice_values == sliced_res[sliceno], "Bad reader hashfilter: slice %d of %d gave %r instead of %r" % (sliceno, slices, slice_values, sliced_res[sliceno],)
+ assert slice_values == sliced_res[sliceno], f"Bad reader hashfilter: slice {sliceno} of {slices} gave {slice_values!r} instead of {sliced_res[sliceno]!r}"
for slices in range(1, 24):
slice_test(slices, False)
slice_test(slices, True)
@@ -222,7 +220,7 @@ def slice_test(slices, spread_None):
fh.write(value)
with r_mk(TMP_FN) as fh:
tmp = list(fh)
- assert tmp == [None, None, None], "Bad spread_None %sfor %d slices" % ("from default " if "default" in kw else "", slices,)
+ assert tmp == [None, None, None], f"Bad spread_None {'from default ' if 'default' in kw else ''}for {slices} slices"
kw["default"] = None
value = object
@@ -243,7 +241,7 @@ def slice_test(slices, spread_None):
print("Hash testing, false things")
for v in (None, "", b"", 0, 0.0, False,):
- assert _dsutil.hash(v) == 0, "%r doesn't hash to 0" % (v,)
+ assert _dsutil.hash(v) == 0, f"{v!r} doesn't hash to 0"
print("Hash testing, strings")
for v in ("", "a", "0", "foo", "a slightly longer string", "\0", "a\0b",):
l_u = _dsutil.WriteUnicode.hash(v)
@@ -252,7 +250,7 @@ def slice_test(slices, spread_None):
u = _dsutil.WriteUnicode.hash(v)
a = _dsutil.WriteAscii.hash(v)
b = _dsutil.WriteBytes.hash(v.encode("utf-8"))
- assert u == l_u == a == l_a == b == l_b, "%r doesn't hash the same" % (v,)
+ assert u == l_u == a == l_a == b == l_b, f"{v!r} doesn't hash the same"
assert _dsutil.hash(b"\xe4") != _dsutil.hash("\xe4"), "Unicode hash fail"
assert _dsutil.WriteBytes.hash(b"\xe4") != _dsutil.WriteUnicode.hash("\xe4"), "Unicode hash fail"
try:
@@ -262,8 +260,8 @@ def slice_test(slices, spread_None):
pass
print("Hash testing, numbers")
for v in (0, 1, 2, 9007199254740991, -42):
- assert _dsutil.WriteInt64.hash(v) == _dsutil.WriteFloat64.hash(float(v)), "%d doesn't hash the same" % (v,)
- assert _dsutil.WriteInt64.hash(v) == _dsutil.WriteNumber.hash(v), "%d doesn't hash the same" % (v,)
+ assert _dsutil.WriteInt64.hash(v) == _dsutil.WriteFloat64.hash(float(v)), f"{v} doesn't hash the same"
+ assert _dsutil.WriteInt64.hash(v) == _dsutil.WriteNumber.hash(v), f"{v} doesn't hash the same"
print("Number boundary test")
Z = 128 * 1024 # the internal buffer size in _dsutil
diff --git a/scripts/templates/a_check.py b/scripts/templates/a_check.py
index d8b42770..5e23d346 100644
--- a/scripts/templates/a_check.py
+++ b/scripts/templates/a_check.py
@@ -4,17 +4,17 @@
options = {'prefix': str, 'version': int}
def check(num, *want):
- job = Job('%s-%d' % (options.prefix, num))
+ job = Job(f'{options.prefix}-{num}')
assert job.params.version == options.version
assert job.params.versions.python_path
if job.params.version > 2:
assert job.params.versions.accelerator
ds = job.dataset()
want_lines = [len(w) for w in want]
- assert ds.lines == want_lines, '%s should have had %r lines but has %r' % (ds, want_lines, ds.lines,)
+ assert ds.lines == want_lines, f'{ds} should have had {want_lines!r} lines but has {ds.lines!r}'
for sliceno, want in enumerate(want):
got = list(ds.iterate(sliceno, ('a', 'b',)))
- assert got == want, '%s slice %d should have had %r but had %r' % (ds, sliceno, want, got,)
+ assert got == want, f'{ds} slice {sliceno} should have had {want!r} but had {got!r}'
def synthesis(job):
check(
@@ -37,9 +37,9 @@ def synthesis(job):
)
# Test for accidental recursion.
sys.setrecursionlimit(49)
- ds51 = Job('%s-%d' % (options.prefix, 51)).dataset()
- assert len(ds51.chain()) == 50, '%s should have had a dataset chain of length 50' % (job,)
+ ds51 = Job(f'{options.prefix}-{51}').dataset()
+ assert len(ds51.chain()) == 50, f'{job} should have had a dataset chain of length 50'
# And check the chain actually contains the expected stuff (i.e. nothing past the first ds).
# (Also to check that iterating the chain doesn't recurse more.)
- ds2 = Job('%s-%d' % (options.prefix, 2)).dataset()
+ ds2 = Job(f'{options.prefix}-{2}').dataset()
assert list(ds2.iterate(None)) == list(ds51.iterate_chain(None))
diff --git a/scripts/templates/a_verify.py b/scripts/templates/a_verify.py
index c4eb2bf7..c6c4a9d0 100644
--- a/scripts/templates/a_verify.py
+++ b/scripts/templates/a_verify.py
@@ -29,9 +29,9 @@ def analysis(sliceno):
good = test_data.sort_data_for_slice(sliceno)
for lineno, got in enumerate(jobs.source.dataset().iterate(sliceno)):
want = next(good)
- assert nanfix(want) == nanfix(got), "Wanted:\n%r\nbut got:\n%r\non line %d in slice %d of %s" % (want, got, lineno, sliceno, jobs.source)
+ assert nanfix(want) == nanfix(got), f"Wanted:\n{want!r}\nbut got:\n{got!r}\non line {lineno} in slice {sliceno} of {jobs.source}"
left_over = len(list(good))
- assert left_over == 0, "Slice %d of %s missing %d lines" % (sliceno, jobs.source, left_over,)
+ assert left_over == 0, f"Slice {sliceno} of {jobs.source} missing {left_over} lines"
if jobs.source.load()['py_version'] > 2 and sys.version_info[0] > 2:
assert list(jobs.source.dataset('pickle').iterate(sliceno, 'p')) == [{'sliceno': sliceno}]
@@ -40,9 +40,9 @@ def synthesis(job):
assert p.versions.accelerator == accelerator.__version__
with job.open_input('proj/accelerator.conf') as fh:
for line in fh:
- if line.startswith('interpreters: p%d ' % (options.n,)):
+ if line.startswith(f'interpreters: p{options.n} '):
path = line.split(' ', 2)[2].strip()[1:-1]
break
else:
- raise Exception('Failed to find interpreter #%d in accelerator.conf' % (options.n,))
+ raise Exception(f'Failed to find interpreter #{options.n} in accelerator.conf')
assert p.versions.python_path == path
diff --git a/setup.py b/setup.py
index 08b52e1e..c159d989 100755
--- a/setup.py
+++ b/setup.py
@@ -20,15 +20,12 @@
# #
############################################################################
-from __future__ import print_function
-
from setuptools import setup, find_packages, Extension
from importlib import import_module
from os.path import exists
import os
from datetime import datetime
from subprocess import check_output, check_call, CalledProcessError
-from io import open
import re
import sys
@@ -79,7 +76,7 @@ def dirty():
assert re.match(r'20\d\d\.\d\d\.\d\d\.(?:dev|rc)\d+$', env_version)
version = env_version
else:
- version = "%s.dev1+%s%s" % (version, commit[:10], dirty(),)
+ version = f"{version}.dev1+{commit[:10]}{dirty()}"
version = version.replace('.0', '.')
with open('accelerator/version.txt', 'w') as fh:
fh.write(version + '\n')
@@ -138,12 +135,8 @@ def method_mod(name):
'bottle>=0.12.7, <0.13; python_version<"3.10"',
'bottle>=0.13, <0.14; python_version>="3.10"',
'waitress>=1.0',
- 'configparser>=3.5.0, <5.0; python_version<"3"',
- 'monotonic>=1.0; python_version<"3"',
- 'selectors2>=2.0; python_version<"3"',
- 'pathlib>=1.0; python_version<"3"',
],
- python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4",
+ python_requires=">=3.6, <4",
ext_modules=[dsutilmodule, dataset_typemodule, csvimportmodule],
@@ -171,7 +164,6 @@ def method_mod(name):
"Operating System :: POSIX",
"Operating System :: POSIX :: BSD :: FreeBSD",
"Operating System :: POSIX :: Linux",
- "Programming Language :: Python :: 2",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: C",