Replace pkg_resources usage

.github/workflows/unittests.yml

@@ -52,10 +52,6 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install setuptools for pkg_resources for Python 3.12+, #2569
-      if: ${{ matrix.python-version >= '3.12' }}
-      run: pip install 'setuptools<82'
-
     - name: Install dependencies for full run
       if: ${{ matrix.type == 'full' }}
       env:

intelmq/bin/intelmqctl.py

@@ -17,7 +17,6 @@
 import traceback
 import time
 
-import pkg_resources
 from ruamel.yaml import YAML
 
 from intelmq import (DEFAULT_LOGGING_LEVEL,  # noqa: F401
@@ -127,8 +126,8 @@ def __init__(self, interactive: bool = False, returntype: ReturnType = ReturnTyp
 
         APPNAME = "intelmqctl"
         try:
-            VERSION = pkg_resources.get_distribution("intelmq").version
-        except pkg_resources.DistributionNotFound:  # pragma: no cover
+            VERSION = utils.package_version("intelmq")
+        except utils.PackageNotFoundError:  # pragma: no cover
             # can only happen in interactive mode
             self._logger.error('No valid IntelMQ installation found: DistributionNotFound')
             sys.exit(1)

intelmq/bin/intelmqsetup.py

@@ -22,7 +22,6 @@
 import shutil
 import stat
 import sys
-import pkg_resources
 
 from grp import getgrnam
 from pathlib import Path
@@ -53,6 +52,7 @@
 from intelmq import (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH,
                      VAR_STATE_PATH, STATE_FILE_PATH)
 from intelmq.bin.intelmqctl import IntelMQController
+from intelmq.lib.utils import package_resource_path, package_version
 
 
 FILE_OUTPUT_PATH = Path(VAR_STATE_PATH) / 'file-output/'
@@ -157,7 +157,7 @@ def intelmqsetup_core(ownership=True, state_file=STATE_FILE_PATH):
     create_directory(DEFAULT_LOGGING_PATH, 0o40755)
     create_directory(CONFIG_DIR, 0o40775)
 
-    example_path = Path(pkg_resources.resource_filename('intelmq', 'etc'))
+    example_path = Path(package_resource_path('intelmq', 'etc'))
     example_confs = [example_path / 'runtime.yaml', example_path / 'harmonization.conf']
     for example_conf in example_confs:
         fname = Path(example_conf).name
@@ -200,7 +200,7 @@ def intelmqsetup_api(ownership: bool = True, webserver_user: Optional[str] = Non
     if ownership:
         change_owner(ETC_INTELMQ_MANAGER, group='intelmq')
 
-    base = Path(pkg_resources.resource_filename('intelmq_api', '')).parent
+    base = Path(package_resource_path('intelmq_api')).parent
     api_config = base / 'etc/intelmq/api-config.json'
     etc_intelmq_config = ETC_INTELMQ / 'api-config.json'
     api_sudoers = base / 'etc/intelmq/api-sudoers.conf'
@@ -249,7 +249,7 @@ def intelmqsetup_api(ownership: bool = True, webserver_user: Optional[str] = Non
 
 def intelmqsetup_api_webserver_configuration(webserver_configuration_directory: Optional[str] = None):
     webserver_configuration_dir = webserver_configuration_directory or find_webserver_configuration_directory()
-    api_config = Path(pkg_resources.resource_filename('intelmq_api', '')).parent / 'etc/intelmq/api-apache.conf'
+    api_config = Path(package_resource_path('intelmq_api')).parent / 'etc/intelmq/api-apache.conf'
     apache_api_config = webserver_configuration_dir / 'api-apache.conf'
     if api_config.exists() and not apache_api_config.exists():
         shutil.copy(api_config, apache_api_config)
@@ -266,9 +266,9 @@ def intelmqsetup_api_webserver_configuration(webserver_configuration_directory:
 
 def intelmqsetup_manager_webserver_configuration(webserver_configuration_directory: Optional[str] = None):
     webserver_configuration_dir = webserver_configuration_directory or find_webserver_configuration_directory()
-    manager_config_1 = Path(pkg_resources.resource_filename('intelmq_manager', '')).parent / 'etc/intelmq/manager-apache.conf'
+    manager_config_1 = Path(package_resource_path('intelmq_manager')).parent / 'etc/intelmq/manager-apache.conf'
     # IntelMQ Manager >= 3.1.0
-    manager_config_2 = Path(pkg_resources.resource_filename('intelmq_manager', '')) / 'manager-apache.conf'
+    manager_config_2 = Path(package_resource_path('intelmq_manager')) / 'manager-apache.conf'
     manager_config = manager_config_2 if manager_config_2.exists() else manager_config_1
     apache_manager_config = webserver_configuration_dir / 'manager-apache.conf'
     if manager_config.exists() and not apache_manager_config.exists():
@@ -287,7 +287,7 @@ def intelmqsetup_manager_generate():
         print('Unable to build intelmq-manager files. Installed version of intelmq-manager is too old, at least version 3.1.0 is required.',
               file=sys.stderr)
         return
-    src_dir = Path(pkg_resources.resource_filename('intelmq_manager', ''))
+    src_dir = Path(package_resource_path('intelmq_manager'))
     html_dir_destination = Path('/usr/share/intelmq_manager/html')
 
     if not src_dir.as_posix().startswith('/usr/'):
@@ -338,7 +338,7 @@ def main():
     else:
         print('Skipping intelmq-manager configuration.')
     if intelmq_manager and not args.skip_manager:
-        manager_version = pkg_resources.get_distribution('intelmq-manager').version
+        manager_version = package_version('intelmq-manager')
         print(f'Generate and save intelmq-manager (version {manager_version}) static files.')
         intelmqsetup_manager_generate()
 

intelmq/bots/collectors/shodan/collector_alert.py

@@ -3,11 +3,12 @@
 SPDX-License-Identifier: AGPL-3.0-or-later
 """
 
-from pkg_resources import get_distribution
 from json import dumps as json_dumps
 from typing import Optional
 
-from intelmq.lib.bot import CollectorBot
+from packaging.version import parse as parse_version
+
+from intelmq.lib.bot import CollectorBot, utils
 
 try:
     from shodan import Shodan
@@ -32,7 +33,7 @@ def init(self):
                        'https': self.https_proxy}
                       if self.https_proxy
                       else {})
-        if tuple(int(v) for v in get_distribution("shodan").version.split('.')) <= (1, 8, 1):
+        if parse_version(utils.package_version("shodan")) <= parse_version("1.8.1"):
             if self.proxy:
                 raise ValueError('Proxies are given but shodan-python > 1.8.1 is needed for proxy support.')
             else:
@@ -59,8 +60,8 @@ def check(parameters: dict) -> Optional[list[list[str]]]:
             else:
                 messages.append(["error", "Library 'shodan' is needed but not installed."])
         else:
-            shodan_version = tuple(int(v) for v in get_distribution("shodan").version.split('.'))
-            if 'https_proxy' in parameters and shodan_version <= (1, 8, 1):
+            shodan_version = parse_version(utils.package_version("shodan"))
+            if 'https_proxy' in parameters and shodan_version <= parse_version("1.8.1"):
                 messages.append(["error", "Library 'shodan' needs to be updated. At least version 1.8.1 is required for HTTPS Proxy support."])
 
         return messages

intelmq/bots/collectors/shodan/collector_stream.py

@@ -11,14 +11,14 @@
 * countries: A list of strings or a comma separated list with country codes
 * alert: An alert ID from monitor.shodan.io
 """
-import pkg_resources
 from http.client import IncompleteRead
 from urllib3.exceptions import ProtocolError, ReadTimeoutError
 
+from packaging.version import parse as parse_version
 from requests.exceptions import ChunkedEncodingError, ConnectionError
 from typing import List, Optional
 
-from intelmq.lib.bot import CollectorBot
+from intelmq.lib.bot import CollectorBot, utils
 
 try:
     import shodan
@@ -38,7 +38,7 @@ def init(self):
             raise ValueError("Library 'shodan' is needed but not installed.")
 
         self.set_request_parameters()
-        if tuple(int(v) for v in pkg_resources.get_distribution("shodan").version.split('.')) <= (1, 8, 1):
+        if parse_version(utils.package_version("shodan")) <= parse_version("1.8.1"):
             if self.proxy:
                 raise ValueError('Proxies are given but shodan-python > 1.8.1 is needed for proxy support.')
             else:

intelmq/bots/parsers/ioc_extractor/parser.py

@@ -21,7 +21,6 @@
     classification_type : string with a valid classificationtype
 """
 import re
-import pkg_resources
 
 from typing import Optional
 from collections.abc import Iterable
@@ -30,6 +29,7 @@
 from intelmq.lib.exceptions import InvalidArgument
 from intelmq.lib.harmonization import ClassificationType
 from intelmq.lib.exceptions import MissingDependencyError
+from packaging.version import parse as parse_version
 
 try:
     from url_normalize import url_normalize
@@ -57,8 +57,8 @@ class IocExtractorParserBot(ParserBot):
     def init(self):
         if url_normalize is None:
             raise MissingDependencyError("url-normalize")
-        url_version = pkg_resources.get_distribution("url-normalize").version
-        if tuple(int(v) for v in url_version.split('.')) < (1, 4, 1) and self.default_scheme is not None:
+        url_version = utils.package_version("url-normalize")
+        if parse_version(url_version) < parse_version("1.4.1") and self.default_scheme is not None:
             raise ValueError("Parameter 'default_scheme' given but 'url-normalize' version %r does not support it. "
                              "Get at least version '1.4.1'." % url_version)
         if get_tld is None:

intelmq/lib/bot.py

@@ -30,7 +30,6 @@
 from copy import deepcopy
 from datetime import datetime, timedelta
 from typing import Any, List, Optional, Union, Tuple
-from pkg_resources import resource_filename
 
 import intelmq.lib.message as libmessage
 from intelmq import (DEFAULT_LOGGING_PATH,
@@ -929,7 +928,9 @@ def __load_harmonization_configuration(self):
             if self._standalone:
                 raise
             else:
-                self._harmonization = utils.load_configuration(resource_filename('intelmq', 'etc/harmonization.conf'))
+                self._harmonization = utils.load_configuration(
+                    utils.package_resource_path('intelmq', 'etc/harmonization.conf')
+                )
 
     def new_event(self, *args, **kwargs):
         return libmessage.Event(*args, harmonization=self.harmonization, **kwargs)

intelmq/lib/message.py

@@ -15,7 +15,6 @@
 from collections import defaultdict
 from typing import Any, Dict, Optional, Union, Tuple
 from collections.abc import Iterable, Sequence
-from pkg_resources import resource_filename
 
 import intelmq.lib.exceptions as exceptions
 import intelmq.lib.harmonization
@@ -113,7 +112,9 @@ def __init__(self, message: Union[dict, tuple] = (), auto: bool = False,
                 harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE)
             except ValueError:
                 # Fallback to internal harmonization file
-                harmonization = utils.load_configuration(resource_filename('intelmq', 'etc/harmonization.conf'))
+                harmonization = utils.load_configuration(
+                    utils.package_resource_path('intelmq', 'etc/harmonization.conf')
+                )
         try:
             self.harmonization_config = harmonization[classname]
         except KeyError:

intelmq/lib/test.py

@@ -18,7 +18,6 @@
 import unittest.mock as mock
 from itertools import chain
 
-import pkg_resources
 import redis
 
 import intelmq.lib.message as message
@@ -66,8 +65,7 @@ def mocked(conf_file):
                              }}
         elif conf_file.startswith(CONFIG_DIR):
             confname = os.path.join('etc/', os.path.split(conf_file)[-1])
-            fname = pkg_resources.resource_filename('intelmq',
-                                                    confname)
+            fname = utils.package_resource_path('intelmq', confname)
             with open(fname) as fpconfig:
                 return json.load(fpconfig)
         else:
@@ -187,8 +185,9 @@ def setUpClass(cls):
         elif cls.use_cache and os.environ.get('INTELMQ_SKIP_REDIS'):
             cls.skipTest(cls, 'Requested cache requires deactivated Redis.')
 
-    harmonization = utils.load_configuration(pkg_resources.resource_filename('intelmq',
-                                                                             'etc/harmonization.conf'))
+    harmonization = utils.load_configuration(
+        utils.package_resource_path('intelmq', 'etc/harmonization.conf')
+    )
 
     def new_report(self, auto=False, examples=False):
         return message.Report(harmonization=self.harmonization, auto=auto)

intelmq/lib/upgrades.py

@@ -4,11 +4,10 @@
 SPDX-License-Identifier: AGPL-3.0-or-later
 """
 from collections import OrderedDict
-from pkg_resources import resource_filename
 from pathlib import Path
 from intelmq import CONFIG_DIR
 
-from intelmq.lib.utils import load_configuration, write_configuration
+from intelmq.lib.utils import load_configuration, package_resource_path, write_configuration
 
 __all__ = ['v100_dev7_modify_syntax',
            'v110_shadowserver_feednames',
@@ -380,8 +379,9 @@ def harmonization(configuration, harmonization, dry_run, **kwargs):
     Checks if all harmonization fields and types are correct
     """
     changed = None
-    original = load_configuration(resource_filename('intelmq',
-                                                    'etc/harmonization.conf'))
+    original = load_configuration(
+        package_resource_path('intelmq', 'etc/harmonization.conf')
+    )
     for msg_type, msg in original.items():
         if msg_type not in harmonization:
             harmonization[msg_type] = msg
@@ -1008,7 +1008,7 @@ def v350_new_fields(configuration, harmonization, dry_run, **kwargs):
         return changed, configuration, harmonization
 
     builtin_harmonisation = load_configuration(
-        resource_filename("intelmq", "etc/harmonization.conf")
+        package_resource_path("intelmq", "etc/harmonization.conf")
     )
     for field in [
         "severity",

intelmq/lib/utils.py

@@ -20,6 +20,7 @@
 import grp
 import gzip
 import importlib
+from importlib import resources
 import inspect
 import io
 import json
@@ -52,17 +53,18 @@
 from intelmq.lib.exceptions import DecodingError
 
 try:
-    from importlib.metadata import entry_points
+    from importlib.metadata import PackageNotFoundError, entry_points, version as package_version
 except ImportError:
-    from importlib_metadata import entry_points
+    from importlib_metadata import PackageNotFoundError, entry_points, version as package_version
 
 
 __all__ = ['base64_decode', 'base64_encode', 'decode', 'encode',
            'load_configuration', 'load_parameters', 'log', 'parse_logline',
            'reverse_readline', 'error_message_from_exc', 'parse_relative',
            'RewindableFileHandle',
            'file_name_from_response',
-           'list_all_bots', 'get_global_settings',
+           'list_all_bots', 'get_global_settings', 'package_resource_path',
+           'package_version',
            ]
 
 # Used loglines format
@@ -230,6 +232,18 @@ def load_configuration(configuration_filepath: str) -> dict:
     return config
 
 
+def package_resource_path(package: str, resource: str = '') -> str:
+    """
+    Return a filesystem path for a resource bundled in an installed package.
+    """
+    package_reference = sys.modules.get(package, package)
+    resource_path = resources.files(package_reference)
+    for part in resource.split('/'):
+        if part:
+            resource_path = resource_path.joinpath(part)
+    return str(resource_path)
+
+
 def write_configuration(configuration_filepath: str,
                         content: dict, backup: bool = True,
                         new=False, useyaml=True) -> Optional[bool]:

intelmq/tests/bin/test_intelmqctl.py

@@ -9,8 +9,6 @@
 from unittest import mock
 from pathlib import Path
 
-from pkg_resources import resource_filename
-
 import intelmq.bin.intelmqctl as ctl
 import intelmq.lib.utils as utils
 from intelmq.lib.test import skip_installation
@@ -90,13 +88,15 @@ def _extend_config(self, path: str, to_extend: dict, useyaml: bool = True):
         utils.write_configuration(path, config, backup=False, useyaml=useyaml)
 
     def _load_default_harmonization(self):
-        default = utils.load_configuration(resource_filename('intelmq',
-                                                             'etc/harmonization.conf'))
+        default = utils.load_configuration(
+            utils.package_resource_path('intelmq', 'etc/harmonization.conf')
+        )
         self._extend_config(self.tmp_harmonization, default, useyaml=False)
 
     def _load_default_runtime(self):
-        default = utils.load_configuration(resource_filename('intelmq',
-                                                             'etc/runtime.yaml'))
+        default = utils.load_configuration(
+            utils.package_resource_path('intelmq', 'etc/runtime.yaml')
+        )
         self._extend_config(self.tmp_runtime, default)
 
     def tearDown(self):

intelmq/tests/bin/test_psql_initdb.py

@@ -14,9 +14,8 @@
 import tempfile
 import unittest
 
-import pkg_resources
-
 import intelmq.bin.intelmq_psql_initdb as psql_initdb
+from intelmq.lib.utils import package_resource_path
 
 
 class TestPsqlInit(unittest.TestCase):
@@ -55,7 +54,7 @@ def test_output(self):
         with open(os.path.join(os.path.dirname(__file__),
                                'initdb.sql')) as handle:
             expected = handle.read()
-        fname = pkg_resources.resource_filename('intelmq', 'etc/harmonization.conf')
+        fname = package_resource_path('intelmq', 'etc/harmonization.conf')
         self.assertEqual(psql_initdb.generate(fname).strip(), expected.strip())
 
     def test_skip_generating_events_table_schema(self):