diff --git a/bart-registrant b/bart-registrant index b8659e7..cf6e520 100755 --- a/bart-registrant +++ b/bart-registrant @@ -60,7 +60,7 @@ DEFAULT_CONFIG_FILE = "/etc/bart/bart.conf" DEFAULT_LOGFILE = "/var/log/bart-registration.log" DEFAULT_HOSTKEY = "/etc/grid-security/hostkey.pem" DEFAULT_HOSTCERT = "/etc/grid-security/hostcert.pem" -DEFAULT_CERTDIR = "/etc/grid-security/certificates" +DEFAULT_CERTDIR = None DEFAULT_LOG_DIR = "/var/spool/bart/usagerecords/" DEFAULT_BATCH_SIZE = 100 DEFAULT_TIMEOUT = "10," @@ -257,7 +257,6 @@ def httpRequest(url, method="GET", payload=None, ctxFactory=None, timeout=None): """ params = { "timeout": timeout, - "verify": False, } if ctxFactory: params["cert"] = (ctxFactory.cert_path, ctxFactory.key_path) diff --git a/bart/__init__.py b/bart/__init__.py index a9eee25..e69de29 100644 --- a/bart/__init__.py +++ b/bart/__init__.py @@ -1,8 +0,0 @@ -# bart/__init__.py - -import time -gmt = time.gmtime() - -# set this to "correct" version when making a release -__version__ = 'svn-%04d%02d%02d' % (gmt.tm_year, gmt.tm_mon, gmt.tm_mday) - diff --git a/bart/config.py b/bart/config.py index c263ad9..7ec197a 100644 --- a/bart/config.py +++ b/bart/config.py @@ -22,7 +22,7 @@ DEFAULT_LOG_FILE = '/var/log/bart-logger.log' DEFAULT_LOG_DIR = '/var/spool/bart/usagerecords' DEFAULT_STATEDIR = '/var/spool/bart' -DEFAULT_SUPPRESS_USERMAP_INFO = 'false' +DEFAULT_SUPPRESS_USERMAP_INFO = 'true' DEFAULT_LOG_LEVEL = 'INFO' DEFAULT_STDERR_LEVEL = None diff --git a/bart/slurm.py b/bart/slurm.py index 60e6880..bb437d9 100644 --- a/bart/slurm.py +++ b/bart/slurm.py @@ -7,6 +7,7 @@ # Author: Magnus Jonsson # Copyright: Nordic Data Grid Facility (2010) +import ast import os import time import datetime @@ -30,7 +31,7 @@ DEFAULT_IDTIMESTAMP = 'true' MAX_DAYS = 'max_days' -MAX_DAYS_DEFAULT = 7 +MAX_DAYS_DEFAULT = 0 # This fills in the "processors" field. PROCESSORS_UNIT = 'processors_unit' @@ -51,6 +52,14 @@ USERS = 'users' USERS_DEFAULT = None +# Filter on account +ACCOUNT_FILTER = 'account_filter' +DEFAULT_ACCOUNT_FILTER = None + +# Map account name +ACCOUNT_MAP = 'account_map' +ACCOUNT_MAP_DEFAULT = None + CONFIG = { STATEFILE: { 'required': False }, STATEFILE_DEFAULT: { 'required': False, type: 'int' }, @@ -60,6 +69,8 @@ CHARGE_UNIT: { 'required': False }, CHARGE_SCALE: { 'required': False, type: 'float' }, USERS: { 'required': False }, + ACCOUNT_FILTER: { 'required': False }, + ACCOUNT_MAP: { 'required': False }, } COMMAND = 'sacct %(users)s --duplicates --parsable2 --format=JobIDRaw,User,Partition,Submit,Start,End,Account,Elapsed,UserCPU,AllocTRES,Nodelist,NNodes --state=%(states)s --starttime="%(starttime)s" --endtime="%(endtime)s"' @@ -90,14 +101,17 @@ def versioncmp(a, b): aa = [ int(x) for x in re.findall(r"\d+", a) ] bb = [ int(x) for x in re.findall(r"\d+", b) ] - for i in range(min(len(aa), len(bb))): + a_length = len(aa) + b_length = len(bb) + + for i in range(min(a_length, b_length)): if aa[i] < bb[i]: return -1 elif aa[i] > bb[i]: return 1 ## If we get here, all common components are equal. Decide by the number of components: - return cmp(a_length, b_length) + return (a_length > b_length) - (a_length < b_length) class SlurmBackend: @@ -114,7 +128,7 @@ def __init__(self, state_starttime, max_days, user_list): while not self.results and croped: # Check if number of days since last run is > search_days, if so only # advance max_days days - search_days += int(max_days) + search_days += max_days if max_days > 0 and datetime.datetime.now() - datetime.datetime.strptime( state_starttime, "%Y-%m-%dT%H:%M:%S" ) > datetime.timedelta(days=search_days): self.end_str = datetime.datetime.strptime( state_starttime, "%Y-%m-%dT%H:%M:%S" ) + datetime.timedelta(days=search_days) self.end_str = self.end_str.isoformat().split('.')[0] @@ -155,6 +169,34 @@ def __init__(self,cfg): self.processors_unit = cfg.getConfigValue(SECTION, PROCESSORS_UNIT, DEFAULT_PROCESSORS_UNIT) self.charge_unit = cfg.getConfigValue(SECTION, CHARGE_UNIT, DEFAULT_CHARGE_UNIT) self.charge_scale = cfg.getConfigValue(SECTION, CHARGE_SCALE, DEFAULT_CHARGE_SCALE) + self.account_filter = cfg.getConfigValue(SECTION, ACCOUNT_FILTER, DEFAULT_ACCOUNT_FILTER) + self.account_map = cfg.getConfigValue(SECTION, ACCOUNT_MAP, ACCOUNT_MAP_DEFAULT) + self.account_rx = None + if self.account_filter: + self.account_rx = re.compile(self.account_filter) + self.account_map_list = None + map_ok = True + if self.account_map: + self.account_map_list = ast.literal_eval(self.account_map) + # Make sure its really a list and that each entry is really a dict + if not isinstance(self.account_map_list, list): + logging.error('ACCOUNT_MAP in config file is not a list: %s' % self.account_map) + map_ok = False + else: + for d in self.account_map_list: + if not isinstance(d, dict): + logging.error('ACCOUNT_MAP item in config file is not a dict: %s' % d) + map_ok = False + else: + if 'regex' not in d: + logging.error('ACCOUNT_MAP item in config file lacks a "regex" key: %s' % d) + map_ok = False + if 'replace' not in d: + logging.error('ACCOUNT_MAP item in config file lacks a "replace" key: %s' % d) + map_ok = False + if not map_ok: + sys.stderr.write('ACCOUNT_MAP in config is incorrectly defined. See the documentation for how to defined it.') + sys.exit(1) def getStateFile(self): return self.cfg.getConfigValue(SECTION, STATEFILE, DEFAULT_STATEFILE) @@ -312,13 +354,38 @@ def createUsageRecord(self, log_entry, hostname, user_map, project_map): return ur + def filter_on_account(self, account): + """ + Filter on account + """ + ret = True + if self.account_rx: + if not self.account_rx.match(account): + ret = False + + return ret + + def rewrite_account_name(self, account_name): + """ + Rewrite account name + """ + if self.account_map_list: + for d in self.account_map_list: + account_name = re.sub(d['regex'], d['replace'], account_name) + + return account_name + def generateUsageRecords(self, hostname, user_map, project_map): """ Starts the UR generation process. """ self.missing_user_mappings = {} - tlp = SlurmBackend(self.state, self.cfg.getConfigValue(SECTION, MAX_DAYS, MAX_DAYS_DEFAULT), self.cfg.getConfigValue(SECTION, USERS, USERS_DEFAULT)) + max_days = int(self.cfg.getConfigValue(SECTION, MAX_DAYS, MAX_DAYS_DEFAULT)) + if max_days == 0: + max_days = (datetime.datetime.now() - datetime.datetime.strptime(self.state, "%Y-%m-%dT%H:%M:%S")).days + 1 + + tlp = SlurmBackend(self.state, max_days, self.cfg.getConfigValue(SECTION, USERS, USERS_DEFAULT)) count = 0 while True: @@ -327,11 +394,13 @@ def generateUsageRecords(self, hostname, user_map, project_map): if log_entry is None: break # no more log entries - ur = self.createUsageRecord(log_entry, hostname, user_map, project_map) - - if ur is not None: - common.writeUr(ur,self.cfg) - count = count + 1 + if self.filter_on_account(log_entry[6]): + log_entry[6] = self.rewrite_account_name(log_entry[6]) + ur = self.createUsageRecord(log_entry, hostname, user_map, project_map) + + if ur is not None: + common.writeUr(ur,self.cfg) + count = count + 1 # only update state if a entry i written if count > 0: @@ -345,7 +414,7 @@ def parseGeneratorState(self,state): This is returns the last jobid processed. """ if state is None or len(state) == 0: - # no statefile -> we start from 50000 (DEFAULT_STATEFILE_DEFAULT) seconds / 5.7 days ago + # no statefile -> we start from STATEFILE_DEFAULT seconds ago sfd = int(self.cfg.getConfigValue(SECTION, STATEFILE_DEFAULT, DEFAULT_STATEFILE_DEFAULT)) dt = datetime.datetime.now()-datetime.timedelta(seconds=sfd) state = dt.isoformat().split('.')[0] diff --git a/bart/usagerecord/urparser.py b/bart/usagerecord/urparser.py index 71dd6d4..b9061ce 100644 --- a/bart/usagerecord/urparser.py +++ b/bart/usagerecord/urparser.py @@ -6,13 +6,14 @@ Copyright: Nordic Data Grid Facility (2010) """ +import logging import time -from twisted.python import log - from bart.ext import isodate from bart.usagerecord import urelements as ur +logger = logging.getLogger(__name__) + # date constants ISO_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" # if we want to convert back some time @@ -26,7 +27,7 @@ def parseBoolean(value): elif value == '0' or value.lower() == 'false': return False else: - log.msg('Failed to parse value %s into boolean' % value, system='sgas.UsageRecord') + logger.info('Failed to parse value %s into boolean' % value) return None @@ -34,7 +35,7 @@ def parseInt(value): try: return int(value) except ValueError: - log.msg("Failed to parse float: %s" % value, system='sgas.UsageRecord') + logger.info("Failed to parse float: %s" % value) return None @@ -42,7 +43,7 @@ def parseFloat(value): try: return float(value) except ValueError: - log.msg("Failed to parse float: %s" % value, system='sgas.UsageRecord') + logger.info("Failed to parse float: %s" % value) return None @@ -51,7 +52,7 @@ def parseISODuration(value): td = isodate.parse_duration(value) return (td.days * 3600*24) + td.seconds # screw microseconds except ValueError: - log.msg("Failed to parse duration: %s" % value, system='sgas.UsageRecord') + logger.info("Failed to parse duration: %s" % value) return None @@ -60,10 +61,10 @@ def parseISODateTime(value): dt = isodate.parse_datetime(value) return time.strftime(JSON_DATETIME_FORMAT, dt.utctimetuple()) except ValueError as e: - log.msg("Failed to parse datetime value: %s (%s)" % (value, str(e)), system='sgas.UsageRecord') + logger.info("Failed to parse datetime value: %s (%s)" % (value, str(e))) return None except isodate.ISO8601Error as e: - log.msg("Failed to parse ISO datetime value: %s (%s)" % (value, str(e)), system='sgas.UsageRecord') + logger.info("Failed to parse ISO datetime value: %s (%s)" % (value, str(e))) return None @@ -139,8 +140,8 @@ def setIfNotNone(key, value): elif element.tag == ur.SUBMIT_TIME: r['submit_time'] = parseISODateTime(element.text) - elif element.tag == ur.KSI2K_WALL_DURATION: log.msg('Got ksi2k wall duration element, ignoring (deprecated)', system='sgas.UsageRecord') - elif element.tag == ur.KSI2K_CPU_DURATION: log.msg('Got ksi2k cpu duration element, ignoring (deprecated)', system='sgas.UsageRecord') + elif element.tag == ur.KSI2K_WALL_DURATION: logger.info('Got ksi2k wall duration element, ignoring (deprecated)') + elif element.tag == ur.KSI2K_CPU_DURATION: logger.info('Got ksi2k cpu duration element, ignoring (deprecated)') elif element.tag == ur.USER_TIME: r['user_time'] = parseISODuration(element.text) elif element.tag == ur.KERNEL_TIME: r['kernel_time'] = parseISODuration(element.text) elif element.tag == ur.EXIT_CODE: r['exit_code'] = parseInt(element.text) @@ -188,7 +189,7 @@ def setIfNotNone(key, value): r.setdefault('uploads', []).append(upload) else: - log.msg("Unhandled UR element: %s" % element.tag, system='sgas.UsageRecord') + logger.info("Unhandled UR element: %s" % element.tag) # backwards logger compatability # alot of loggers set node_count when they should have used processors, therefore: diff --git a/bart/usagerecord/usagerecord.py b/bart/usagerecord/usagerecord.py index 67df05a..ce98331 100644 --- a/bart/usagerecord/usagerecord.py +++ b/bart/usagerecord/usagerecord.py @@ -7,8 +7,8 @@ # Author: Henrik Thostrup Jensen # Copyright: Nordic Data Grid Facility (2009, 2010) +import importlib.metadata import time -from bart import __version__ from bart.usagerecord import urelements as ur @@ -25,7 +25,10 @@ # values for the logger name + version LOGGER_NAME_VALUE = 'SGAS-BaRT' -LOGGER_VERSION_VALUE = __version__ +try: + LOGGER_VERSION_VALUE = importlib.metadata.version("sgas-bart") +except importlib.metadata.PackageNotFoundError: + LOGGER_VERSION_VALUE = "unknown" # register namespaces in element tree so we get more readable xml files # the semantics of the xml files does not change due to this diff --git a/bart/usagerecord/verify.py b/bart/usagerecord/verify.py index 361fbfe..9a2b212 100644 --- a/bart/usagerecord/verify.py +++ b/bart/usagerecord/verify.py @@ -6,19 +6,19 @@ # Author: Magnus Jonsson # Copyright: NeIC 2014 - +import logging from bart.usagerecord import urparser -from twisted.python import log +logger = logging.getLogger(__name__) def verify(ur): try: d = urparser.xmlToDict(ur) if d['record_id'] is None: - log.err("No record_id found") + logger.error("No record_id found") return False except: - log.err("Failed to convert UR XML into dict") + logger.error("Failed to convert UR XML into dict") return False return True diff --git a/debian/changelog b/debian/changelog index fde0cf1..bf26c0d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +sgas-bart (008-hpc2n0.1) focal; urgency=medium + + * Added account filter and map functionality + + -- Åke Sandgren Thu, 15 Feb 2024 10:54:51 +0100 + sgas-bart (007-1) unstable; urgency=low * se Changelog for changes... diff --git a/debian/compat b/debian/compat index 7f8f011..48082f7 100644 --- a/debian/compat +++ b/debian/compat @@ -1 +1 @@ -7 +12 diff --git a/debian/control b/debian/control index 5b6ed1b..7b9977d 100644 --- a/debian/control +++ b/debian/control @@ -1,12 +1,14 @@ Source: sgas-bart -Maintainer: Magnus Jonsson +Maintainer: Åke Sandgren Section: python Priority: optional -Build-Depends: python (>= 2.6.5), debhelper (>= 7.4.3) +Build-Depends: python3 (>= 3.8), debhelper (>= 12), dh-python Standards-Version: 3.9.1 Package: sgas-bart Architecture: all -Depends: ${misc:Depends}, python (>= 2.6.5), python-twisted-core (>= 10.0.0), python-twisted-web (>= 10.0.0), python-openssl (>= 0.10), python-dateutil (>= 1.4.1) +Depends: ${misc:Depends}, python3 (>= 3.8), python3-twisted, python3-openssl, python3-dateutil +Replaces: python3-sgas-bart (<= 008-sams-20221101-1) +Conflicts: python3-sgas-bart (<= 008-sams-20221101-1) Description: SGAS Batch system Reporting Tool diff --git a/debian/rules b/debian/rules index 961b8b0..11cd9df 100755 --- a/debian/rules +++ b/debian/rules @@ -1,7 +1,4 @@ #!/usr/bin/make -f %: - # for precise - # dh $@ --with python2 --buildsystem=python_distutils - # for lucid (might also work with precise) - dh $@ + dh $@ --buildsystem=pybuild diff --git a/docs/backend.slurm b/docs/backend.slurm index f7e403e..dabdcb7 100644 --- a/docs/backend.slurm +++ b/docs/backend.slurm @@ -20,8 +20,8 @@ idtimestamp: default=true Adds a timestamp to the recordid to be sure that recordid is unique if Slurm reuses the jobid. -max_days: default=7 -Max number of days to process for every run of bart. +max_days: default=0 +Max number of days to process for every run of bart, or 0 for number of days since last run. processors_unit: default=cpu Which element of the AllocTRES should be used as the PROCESSORS ("number of @@ -34,3 +34,19 @@ If Slurm TRESWeights are used, one typically want to use the "billing" field her charge_scale: default=1.0 Slurm only allows for integer values in "billing", thus you may have needed to scale it up tresweights. The reported charge value will be multiplied by this scale. + +account_filter: default=none +A python regex to filter on specific accounts from the sacct output. +Example, note that there are no ' or " around this: +account_filter=^.*-lm$ + +account_map: default=none +Mapping from slurm account names to the upstream name, i.e. the +usagerecord receiving entity, for instance SUPR. +A python list expression with dictionary elements. The dictionary +elements must contain two key/value pairs, 'regex', and 'replace'. For +example list this (and it can be multiline like in this example): +account_map=[ + {'regex': r'-(lm|gpu)$', 'replace': ''}, + {'regex': r'^snic(\d+)-(\d+)-(\d+)$', 'replace': r'SNIC \1/\2-\3'}, + ] diff --git a/docs/setup b/docs/setup index eaf4902..8683a41 100644 --- a/docs/setup +++ b/docs/setup @@ -14,15 +14,15 @@ are daemons. == Requirements == * Python 2.4 or later -* Twisted Core and Web (http://twistedmatrix.com/) +* Twisted Core (http://twistedmatrix.com/) * PyOpenSSL (https://launchpad.net/pyopenssl) * ElementTree (http://effbot.org/zone/element-index.htm - only needed with Python 2.4) * Python dateutil -Debian/Ubuntu package names: python-twisted python-twisted-web python-openssl +Debian/Ubuntu package names: python-twisted python-openssl python-dateutil (python-elementtree) -CentOS/RedHat package names: python-twisted-core python-twisted-web pyOpenSSL +CentOS/RedHat package names: python-twisted-core pyOpenSSL python-dateutil (python-elementtree) == Installation == diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..36de028 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,4 @@ +# Just enough pyproject.toml to allow building without distutils. +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 6703802..7dbfd0f 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,10 @@ from distutils.command.install import install from distutils.command.install_data import install_data -from bart import __version__ +import time +gmt = time.gmtime() + +version = '%04d%02d%02d' % (gmt.tm_year, gmt.tm_mon, gmt.tm_mday) # nasty global for relocation @@ -49,7 +52,7 @@ def finalize_options(self): setup(name='sgas-bart', - version=__version__, + version=version, description='SGAS Batch system Reporting Tool', author='Henrik Thostrup Jensen / Magnus Jonsson', author_email='magnus@hpc2n.umu.se', @@ -57,6 +60,7 @@ def finalize_options(self): packages=['bart','bart.usagerecord', 'bart.ext', 'bart.ext.isodate'], scripts = ['bart-logger', 'bart-registrant'], + install_requires = ['requests'], cmdclass = cmdclasses, data_files = [ diff --git a/sgas-bart.spec b/sgas-bart.spec index 08902fb..21eccc6 100644 --- a/sgas-bart.spec +++ b/sgas-bart.spec @@ -14,7 +14,7 @@ Prefix: %{_prefix} BuildArch: noarch Vendor: Magnus Jonsson Url: http://www.sgas.se/ -Requires: python-twisted-core, python-twisted-web, pyOpenSSL, python-dateutil +Requires: python-twisted-core, pyOpenSSL, python-dateutil %description Tool for generating usage records from LRMS logs and registering the records to SGAS.