slurm.py: Fix comment about missing state file

bart-registrant

@@ -60,7 +60,7 @@ DEFAULT_CONFIG_FILE = "/etc/bart/bart.conf"
 DEFAULT_LOGFILE = "/var/log/bart-registration.log"
 DEFAULT_HOSTKEY = "/etc/grid-security/hostkey.pem"
 DEFAULT_HOSTCERT = "/etc/grid-security/hostcert.pem"
-DEFAULT_CERTDIR = "/etc/grid-security/certificates"
+DEFAULT_CERTDIR = None
 DEFAULT_LOG_DIR = "/var/spool/bart/usagerecords/"
 DEFAULT_BATCH_SIZE = 100
 DEFAULT_TIMEOUT = "10,"
@@ -257,7 +257,6 @@ def httpRequest(url, method="GET", payload=None, ctxFactory=None, timeout=None):
     """
     params = {
         "timeout": timeout,
-        "verify": False,
     }
     if ctxFactory:
         params["cert"] = (ctxFactory.cert_path, ctxFactory.key_path)

bart/__init__.py

@@ -1,8 +0,0 @@
-# bart/__init__.py
-
-import time
-gmt = time.gmtime()
-
-# set this to "correct" version when making a release
-__version__ = 'svn-%04d%02d%02d' % (gmt.tm_year, gmt.tm_mon, gmt.tm_mday)
-

bart/config.py

@@ -22,7 +22,7 @@
 DEFAULT_LOG_FILE        = '/var/log/bart-logger.log'
 DEFAULT_LOG_DIR         = '/var/spool/bart/usagerecords'
 DEFAULT_STATEDIR        = '/var/spool/bart'
-DEFAULT_SUPPRESS_USERMAP_INFO = 'false'
+DEFAULT_SUPPRESS_USERMAP_INFO = 'true'
 DEFAULT_LOG_LEVEL       = 'INFO'
 DEFAULT_STDERR_LEVEL    = None
 

bart/slurm.py

@@ -7,6 +7,7 @@
 # Author: Magnus Jonsson <magnus@hpc2n.umu.se>
 # Copyright: Nordic Data Grid Facility (2010)
 
+import ast
 import os
 import time
 import datetime
@@ -30,7 +31,7 @@
 DEFAULT_IDTIMESTAMP = 'true'
 
 MAX_DAYS = 'max_days'
-MAX_DAYS_DEFAULT = 7
+MAX_DAYS_DEFAULT = 0
 
 # This fills in the "processors" field.
 PROCESSORS_UNIT         = 'processors_unit'
@@ -51,6 +52,14 @@
 USERS = 'users'
 USERS_DEFAULT = None
 
+# Filter on account
+ACCOUNT_FILTER = 'account_filter'
+DEFAULT_ACCOUNT_FILTER = None
+
+# Map account name
+ACCOUNT_MAP = 'account_map'
+ACCOUNT_MAP_DEFAULT = None
+
 CONFIG = {
             STATEFILE:         { 'required': False },
             STATEFILE_DEFAULT: { 'required': False, type: 'int' },
@@ -60,6 +69,8 @@
             CHARGE_UNIT:       { 'required': False },
             CHARGE_SCALE:      { 'required': False, type: 'float' },
             USERS:             { 'required': False },
+            ACCOUNT_FILTER:    { 'required': False },
+            ACCOUNT_MAP:       { 'required': False },
           }
 
 COMMAND = 'sacct %(users)s --duplicates --parsable2 --format=JobIDRaw,User,Partition,Submit,Start,End,Account,Elapsed,UserCPU,AllocTRES,Nodelist,NNodes --state=%(states)s --starttime="%(starttime)s" --endtime="%(endtime)s"'
@@ -90,14 +101,17 @@ def versioncmp(a, b):
     aa = [ int(x) for x in re.findall(r"\d+", a) ]
     bb = [ int(x) for x in re.findall(r"\d+", b) ]
 
-    for i in range(min(len(aa), len(bb))):
+    a_length = len(aa)
+    b_length = len(bb)
+
+    for i in range(min(a_length, b_length)):
         if aa[i] < bb[i]:
             return -1
         elif aa[i] > bb[i]:
             return 1
 
     ## If we get here, all common components are equal. Decide by the number of components:
-    return cmp(a_length, b_length)
+    return (a_length > b_length) - (a_length < b_length)
 
 
 class SlurmBackend:
@@ -114,7 +128,7 @@ def __init__(self, state_starttime, max_days, user_list):
         while not self.results and croped:
             # Check if number of days since last run is > search_days, if so only
             # advance max_days days        
-            search_days += int(max_days)
+            search_days += max_days
             if max_days > 0 and datetime.datetime.now() - datetime.datetime.strptime( state_starttime, "%Y-%m-%dT%H:%M:%S" ) > datetime.timedelta(days=search_days):
                 self.end_str = datetime.datetime.strptime( state_starttime, "%Y-%m-%dT%H:%M:%S" ) + datetime.timedelta(days=search_days)
                 self.end_str = self.end_str.isoformat().split('.')[0]
@@ -155,6 +169,34 @@ def __init__(self,cfg):
         self.processors_unit = cfg.getConfigValue(SECTION, PROCESSORS_UNIT, DEFAULT_PROCESSORS_UNIT)
         self.charge_unit = cfg.getConfigValue(SECTION, CHARGE_UNIT, DEFAULT_CHARGE_UNIT)
         self.charge_scale = cfg.getConfigValue(SECTION, CHARGE_SCALE, DEFAULT_CHARGE_SCALE)
+        self.account_filter = cfg.getConfigValue(SECTION, ACCOUNT_FILTER, DEFAULT_ACCOUNT_FILTER)
+        self.account_map = cfg.getConfigValue(SECTION, ACCOUNT_MAP, ACCOUNT_MAP_DEFAULT)
+        self.account_rx = None
+        if self.account_filter:
+            self.account_rx = re.compile(self.account_filter)
+        self.account_map_list = None
+        map_ok = True
+        if self.account_map:
+            self.account_map_list = ast.literal_eval(self.account_map)
+            # Make sure its really a list and that each entry is really a dict
+            if not isinstance(self.account_map_list, list):
+                logging.error('ACCOUNT_MAP in config file is not a list: %s' % self.account_map)
+                map_ok = False
+            else:
+                for d in self.account_map_list:
+                    if not isinstance(d, dict):
+                        logging.error('ACCOUNT_MAP item in config file is not a dict: %s' % d)
+                        map_ok = False
+                    else:
+                        if 'regex' not in d:
+                            logging.error('ACCOUNT_MAP item in config file lacks a "regex" key: %s' % d)
+                            map_ok = False
+                        if 'replace' not in d:
+                            logging.error('ACCOUNT_MAP item in config file lacks a "replace" key: %s' % d)
+                            map_ok = False
+        if not map_ok:
+            sys.stderr.write('ACCOUNT_MAP in config is incorrectly defined. See the documentation for how to defined it.')
+            sys.exit(1)
 
     def getStateFile(self):
         return self.cfg.getConfigValue(SECTION, STATEFILE, DEFAULT_STATEFILE)
@@ -312,13 +354,38 @@ def createUsageRecord(self, log_entry, hostname, user_map, project_map):
 
         return ur
 
+    def filter_on_account(self, account):
+        """
+        Filter on account
+        """
+        ret = True
+        if self.account_rx:
+            if not self.account_rx.match(account):
+                ret = False
+
+        return ret
+
+    def rewrite_account_name(self, account_name):
+        """
+        Rewrite account name
+        """
+        if self.account_map_list:
+            for d in self.account_map_list:
+                account_name = re.sub(d['regex'], d['replace'], account_name)
+
+        return account_name
+
     def generateUsageRecords(self, hostname, user_map, project_map):
         """
         Starts the UR generation process.
         """
         self.missing_user_mappings = {}
 
-        tlp = SlurmBackend(self.state, self.cfg.getConfigValue(SECTION, MAX_DAYS, MAX_DAYS_DEFAULT), self.cfg.getConfigValue(SECTION, USERS, USERS_DEFAULT))
+        max_days = int(self.cfg.getConfigValue(SECTION, MAX_DAYS, MAX_DAYS_DEFAULT))
+        if max_days == 0:
+            max_days = (datetime.datetime.now() - datetime.datetime.strptime(self.state, "%Y-%m-%dT%H:%M:%S")).days + 1
+
+        tlp = SlurmBackend(self.state, max_days, self.cfg.getConfigValue(SECTION, USERS, USERS_DEFAULT))
 
         count = 0
         while True:
@@ -327,11 +394,13 @@ def generateUsageRecords(self, hostname, user_map, project_map):
             if log_entry is None:
                 break # no more log entries
 
-            ur = self.createUsageRecord(log_entry, hostname, user_map, project_map)
-
-            if ur is not None: 
-                common.writeUr(ur,self.cfg)            
-                count = count + 1
+            if self.filter_on_account(log_entry[6]):
+                log_entry[6] = self.rewrite_account_name(log_entry[6])
+                ur = self.createUsageRecord(log_entry, hostname, user_map, project_map)
+
+                if ur is not None: 
+                    common.writeUr(ur,self.cfg)            
+                    count = count + 1
 
         # only update state if a entry i written
         if count > 0:
@@ -345,7 +414,7 @@ def parseGeneratorState(self,state):
         This is returns the last jobid processed.
         """
         if state is None or len(state) == 0:
-            # no statefile -> we start from 50000 (DEFAULT_STATEFILE_DEFAULT) seconds / 5.7 days ago
+            # no statefile -> we start from STATEFILE_DEFAULT seconds ago
             sfd = int(self.cfg.getConfigValue(SECTION, STATEFILE_DEFAULT, DEFAULT_STATEFILE_DEFAULT))
             dt = datetime.datetime.now()-datetime.timedelta(seconds=sfd)
             state = dt.isoformat().split('.')[0]

bart/usagerecord/urparser.py

@@ -6,13 +6,14 @@
 Copyright: Nordic Data Grid Facility (2010)
 """
 
+import logging
 import time
 
-from twisted.python import log
-
 from bart.ext import isodate
 from bart.usagerecord import urelements as ur
 
+logger = logging.getLogger(__name__)
+
 
 # date constants
 ISO_TIME_FORMAT   = "%Y-%m-%dT%H:%M:%SZ" # if we want to convert back some time
@@ -26,23 +27,23 @@ def parseBoolean(value):
     elif value == '0' or value.lower() == 'false':
         return False
     else:
-        log.msg('Failed to parse value %s into boolean' % value, system='sgas.UsageRecord')
+        logger.info('Failed to parse value %s into boolean' % value)
         return None
 
 
 def parseInt(value):
     try:
         return int(value)
     except ValueError:
-        log.msg("Failed to parse float: %s" % value, system='sgas.UsageRecord')
+        logger.info("Failed to parse float: %s" % value)
         return None
 
 
 def parseFloat(value):
     try:
         return float(value)
     except ValueError:
-        log.msg("Failed to parse float: %s" % value, system='sgas.UsageRecord')
+        logger.info("Failed to parse float: %s" % value)
         return None
 
 
@@ -51,7 +52,7 @@ def parseISODuration(value):
         td = isodate.parse_duration(value)
         return (td.days * 3600*24) + td.seconds # screw microseconds
     except ValueError:
-        log.msg("Failed to parse duration: %s" % value, system='sgas.UsageRecord')
+        logger.info("Failed to parse duration: %s" % value)
         return None
 
 
@@ -60,10 +61,10 @@ def parseISODateTime(value):
         dt = isodate.parse_datetime(value)
         return time.strftime(JSON_DATETIME_FORMAT, dt.utctimetuple())
     except ValueError as e:
-        log.msg("Failed to parse datetime value: %s (%s)" % (value, str(e)), system='sgas.UsageRecord')
+        logger.info("Failed to parse datetime value: %s (%s)" % (value, str(e)))
         return None
     except isodate.ISO8601Error as e:
-        log.msg("Failed to parse ISO datetime value: %s (%s)" % (value, str(e)), system='sgas.UsageRecord')
+        logger.info("Failed to parse ISO datetime value: %s (%s)" % (value, str(e)))
         return None
 
 
@@ -139,8 +140,8 @@ def setIfNotNone(key, value):
 
         elif element.tag == ur.SUBMIT_TIME:         r['submit_time']    = parseISODateTime(element.text)
 
-        elif element.tag == ur.KSI2K_WALL_DURATION: log.msg('Got ksi2k wall duration element, ignoring (deprecated)', system='sgas.UsageRecord')
-        elif element.tag == ur.KSI2K_CPU_DURATION:  log.msg('Got ksi2k cpu duration element, ignoring (deprecated)', system='sgas.UsageRecord')
+        elif element.tag == ur.KSI2K_WALL_DURATION: logger.info('Got ksi2k wall duration element, ignoring (deprecated)')
+        elif element.tag == ur.KSI2K_CPU_DURATION:  logger.info('Got ksi2k cpu duration element, ignoring (deprecated)')
         elif element.tag == ur.USER_TIME:           r['user_time']           = parseISODuration(element.text)
         elif element.tag == ur.KERNEL_TIME:         r['kernel_time']         = parseISODuration(element.text)
         elif element.tag == ur.EXIT_CODE:           r['exit_code']           = parseInt(element.text)
@@ -188,7 +189,7 @@ def setIfNotNone(key, value):
                     r.setdefault('uploads', []).append(upload)
 
         else:
-            log.msg("Unhandled UR element: %s" % element.tag, system='sgas.UsageRecord')
+            logger.info("Unhandled UR element: %s" % element.tag)
 
     # backwards logger compatability
     # alot of loggers set node_count when they should have used processors, therefore:

bart/usagerecord/usagerecord.py

@@ -7,8 +7,8 @@
 # Author: Henrik Thostrup Jensen <htj@ndgf.org>
 # Copyright: Nordic Data Grid Facility (2009, 2010)
 
+import importlib.metadata
 import time
-from bart import __version__
 
 from bart.usagerecord import urelements as ur
 
@@ -25,7 +25,10 @@
 
 # values for the logger name + version
 LOGGER_NAME_VALUE    = 'SGAS-BaRT'
-LOGGER_VERSION_VALUE = __version__
+try:
+    LOGGER_VERSION_VALUE = importlib.metadata.version("sgas-bart")
+except importlib.metadata.PackageNotFoundError:
+    LOGGER_VERSION_VALUE = "unknown"
 
 # register namespaces in element tree so we get more readable xml files
 # the semantics of the xml files does not change due to this

bart/usagerecord/verify.py

@@ -6,19 +6,19 @@
 # Author: Magnus Jonsson <magnus@hpc2n.umu.se>
 # Copyright: NeIC 2014
 
-
+import logging
 from bart.usagerecord import urparser
 
-from twisted.python import log
+logger = logging.getLogger(__name__)
 
 def verify(ur):
     try:
         d = urparser.xmlToDict(ur)
         if d['record_id'] is None:
-            log.err("No record_id found")
+            logger.error("No record_id found")
             return False
     except:
-        log.err("Failed to convert UR XML into dict")
+        logger.error("Failed to convert UR XML into dict")
         return False
     return True
 

debian/changelog

@@ -1,3 +1,9 @@
+sgas-bart (008-hpc2n0.1) focal; urgency=medium
+
+  * Added account filter and map functionality
+
+ -- Åke Sandgren <ake@hpc2n.umu.se>  Thu, 15 Feb 2024 10:54:51 +0100
+
 sgas-bart (007-1) unstable; urgency=low
 
   * se Changelog for changes...

debian/compat

@@ -1 +1 @@
-7
+12

debian/control

@@ -1,12 +1,14 @@
 Source: sgas-bart
-Maintainer: Magnus Jonsson <magnus@hpc2n.umu.se>
+Maintainer: Åke Sandgren <ake@hpc2n.umu.se>
 Section: python
 Priority: optional
-Build-Depends: python (>= 2.6.5), debhelper (>= 7.4.3)
+Build-Depends: python3 (>= 3.8), debhelper (>= 12), dh-python
 Standards-Version: 3.9.1
 
 Package: sgas-bart
 Architecture: all
-Depends: ${misc:Depends}, python (>= 2.6.5), python-twisted-core (>= 10.0.0), python-twisted-web (>= 10.0.0), python-openssl (>= 0.10), python-dateutil (>= 1.4.1)
+Depends: ${misc:Depends}, python3 (>= 3.8), python3-twisted, python3-openssl, python3-dateutil
+Replaces: python3-sgas-bart (<= 008-sams-20221101-1)
+Conflicts: python3-sgas-bart (<= 008-sams-20221101-1)
 Description: SGAS Batch system Reporting Tool
 

debian/rules

@@ -1,7 +1,4 @@
 #!/usr/bin/make -f
 
 %:
-	# for precise
-	# dh $@ --with python2 --buildsystem=python_distutils
-	# for lucid (might also work with precise)
-	dh $@ 
+	dh $@ --buildsystem=pybuild