From 44a75e9b851505b4a2cb691abeffc1cc566d7a6a Mon Sep 17 00:00:00 2001 From: Mirochill <200482516+Mirochill@users.noreply.github.com> Date: Sat, 23 May 2026 00:29:52 +0200 Subject: [PATCH 1/2] Replace pkg_resources usage Use importlib.resources for bundled configuration and test data paths and importlib.metadata for installed package versions. This removes the runtime dependency on pkg_resources while keeping the existing package data lookup flow centralized in intelmq.lib.utils. The Shodan and url-normalize version checks now use packaging.version.parse instead of tuple splitting, which preserves the existing comparisons and handles normal Python package versions more safely. The temporary CI step that installs old setuptools for #2569 is no longer needed. --- .github/workflows/unittests.yml | 4 ---- intelmq/bin/intelmqctl.py | 5 ++--- intelmq/bin/intelmqsetup.py | 16 +++++++-------- .../bots/collectors/shodan/collector_alert.py | 11 +++++----- .../collectors/shodan/collector_stream.py | 6 +++--- intelmq/bots/parsers/ioc_extractor/parser.py | 6 +++--- intelmq/lib/bot.py | 5 +++-- intelmq/lib/message.py | 5 +++-- intelmq/lib/test.py | 9 ++++----- intelmq/lib/upgrades.py | 9 ++++----- intelmq/lib/utils.py | 19 +++++++++++++++--- intelmq/tests/bin/test_intelmqctl.py | 12 +++++------ intelmq/tests/bin/test_psql_initdb.py | 5 ++--- .../bots/experts/asn_lookup/test_expert.py | 5 ++--- .../tests/bots/experts/fake/test_expert.py | 7 +++---- .../tests/bots/experts/jinja/test_expert.py | 2 -- .../tests/bots/experts/modify/test_expert.py | 20 +++++++++---------- .../recordedfuture_iprisk/test_expert.py | 4 ++-- .../bots/experts/tor_nodes/test_expert.py | 5 ++--- .../bots/parsers/hibp/test_parser_callback.py | 7 +++---- intelmq/tests/lib/test_message.py | 8 +++----- intelmq/tests/lib/test_upgrades.py | 7 +++---- intelmq/tests/lib/test_utils.py | 3 +-- intelmq/tests/test_conf.py | 10 ++++------ 24 files changed, 92 insertions(+), 98 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index a7c1ebc29a..d2ba6dae4f 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -52,10 +52,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install setuptools for pkg_resources for Python 3.12+, #2569 - if: ${{ matrix.python-version >= '3.12' }} - run: pip install 'setuptools<82' - - name: Install dependencies for full run if: ${{ matrix.type == 'full' }} env: diff --git a/intelmq/bin/intelmqctl.py b/intelmq/bin/intelmqctl.py index 9668e0327c..0f4e9f4390 100644 --- a/intelmq/bin/intelmqctl.py +++ b/intelmq/bin/intelmqctl.py @@ -17,7 +17,6 @@ import traceback import time -import pkg_resources from ruamel.yaml import YAML from intelmq import (DEFAULT_LOGGING_LEVEL, # noqa: F401 @@ -127,8 +126,8 @@ def __init__(self, interactive: bool = False, returntype: ReturnType = ReturnTyp APPNAME = "intelmqctl" try: - VERSION = pkg_resources.get_distribution("intelmq").version - except pkg_resources.DistributionNotFound: # pragma: no cover + VERSION = utils.package_version("intelmq") + except utils.PackageNotFoundError: # pragma: no cover # can only happen in interactive mode self._logger.error('No valid IntelMQ installation found: DistributionNotFound') sys.exit(1) diff --git a/intelmq/bin/intelmqsetup.py b/intelmq/bin/intelmqsetup.py index 5d4f65d7c0..5b7a1d61c4 100755 --- a/intelmq/bin/intelmqsetup.py +++ b/intelmq/bin/intelmqsetup.py @@ -22,7 +22,6 @@ import shutil import stat import sys -import pkg_resources from grp import getgrnam from pathlib import Path @@ -53,6 +52,7 @@ from intelmq import (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH, VAR_STATE_PATH, STATE_FILE_PATH) from intelmq.bin.intelmqctl import IntelMQController +from intelmq.lib.utils import package_resource_path, package_version FILE_OUTPUT_PATH = Path(VAR_STATE_PATH) / 'file-output/' @@ -157,7 +157,7 @@ def intelmqsetup_core(ownership=True, state_file=STATE_FILE_PATH): create_directory(DEFAULT_LOGGING_PATH, 0o40755) create_directory(CONFIG_DIR, 0o40775) - example_path = Path(pkg_resources.resource_filename('intelmq', 'etc')) + example_path = Path(package_resource_path('intelmq', 'etc')) example_confs = [example_path / 'runtime.yaml', example_path / 'harmonization.conf'] for example_conf in example_confs: fname = Path(example_conf).name @@ -200,7 +200,7 @@ def intelmqsetup_api(ownership: bool = True, webserver_user: Optional[str] = Non if ownership: change_owner(ETC_INTELMQ_MANAGER, group='intelmq') - base = Path(pkg_resources.resource_filename('intelmq_api', '')).parent + base = Path(package_resource_path('intelmq_api')).parent api_config = base / 'etc/intelmq/api-config.json' etc_intelmq_config = ETC_INTELMQ / 'api-config.json' api_sudoers = base / 'etc/intelmq/api-sudoers.conf' @@ -249,7 +249,7 @@ def intelmqsetup_api(ownership: bool = True, webserver_user: Optional[str] = Non def intelmqsetup_api_webserver_configuration(webserver_configuration_directory: Optional[str] = None): webserver_configuration_dir = webserver_configuration_directory or find_webserver_configuration_directory() - api_config = Path(pkg_resources.resource_filename('intelmq_api', '')).parent / 'etc/intelmq/api-apache.conf' + api_config = Path(package_resource_path('intelmq_api')).parent / 'etc/intelmq/api-apache.conf' apache_api_config = webserver_configuration_dir / 'api-apache.conf' if api_config.exists() and not apache_api_config.exists(): shutil.copy(api_config, apache_api_config) @@ -266,9 +266,9 @@ def intelmqsetup_api_webserver_configuration(webserver_configuration_directory: def intelmqsetup_manager_webserver_configuration(webserver_configuration_directory: Optional[str] = None): webserver_configuration_dir = webserver_configuration_directory or find_webserver_configuration_directory() - manager_config_1 = Path(pkg_resources.resource_filename('intelmq_manager', '')).parent / 'etc/intelmq/manager-apache.conf' + manager_config_1 = Path(package_resource_path('intelmq_manager')).parent / 'etc/intelmq/manager-apache.conf' # IntelMQ Manager >= 3.1.0 - manager_config_2 = Path(pkg_resources.resource_filename('intelmq_manager', '')) / 'manager-apache.conf' + manager_config_2 = Path(package_resource_path('intelmq_manager')) / 'manager-apache.conf' manager_config = manager_config_2 if manager_config_2.exists() else manager_config_1 apache_manager_config = webserver_configuration_dir / 'manager-apache.conf' if manager_config.exists() and not apache_manager_config.exists(): @@ -287,7 +287,7 @@ def intelmqsetup_manager_generate(): print('Unable to build intelmq-manager files. Installed version of intelmq-manager is too old, at least version 3.1.0 is required.', file=sys.stderr) return - src_dir = Path(pkg_resources.resource_filename('intelmq_manager', '')) + src_dir = Path(package_resource_path('intelmq_manager')) html_dir_destination = Path('/usr/share/intelmq_manager/html') if not src_dir.as_posix().startswith('/usr/'): @@ -338,7 +338,7 @@ def main(): else: print('Skipping intelmq-manager configuration.') if intelmq_manager and not args.skip_manager: - manager_version = pkg_resources.get_distribution('intelmq-manager').version + manager_version = package_version('intelmq-manager') print(f'Generate and save intelmq-manager (version {manager_version}) static files.') intelmqsetup_manager_generate() diff --git a/intelmq/bots/collectors/shodan/collector_alert.py b/intelmq/bots/collectors/shodan/collector_alert.py index 7b7fde26ea..d0cce66a2e 100644 --- a/intelmq/bots/collectors/shodan/collector_alert.py +++ b/intelmq/bots/collectors/shodan/collector_alert.py @@ -3,11 +3,12 @@ SPDX-License-Identifier: AGPL-3.0-or-later """ -from pkg_resources import get_distribution from json import dumps as json_dumps from typing import Optional -from intelmq.lib.bot import CollectorBot +from packaging.version import parse as parse_version + +from intelmq.lib.bot import CollectorBot, utils try: from shodan import Shodan @@ -32,7 +33,7 @@ def init(self): 'https': self.https_proxy} if self.https_proxy else {}) - if tuple(int(v) for v in get_distribution("shodan").version.split('.')) <= (1, 8, 1): + if parse_version(utils.package_version("shodan")) <= parse_version("1.8.1"): if self.proxy: raise ValueError('Proxies are given but shodan-python > 1.8.1 is needed for proxy support.') else: @@ -59,8 +60,8 @@ def check(parameters: dict) -> Optional[list[list[str]]]: else: messages.append(["error", "Library 'shodan' is needed but not installed."]) else: - shodan_version = tuple(int(v) for v in get_distribution("shodan").version.split('.')) - if 'https_proxy' in parameters and shodan_version <= (1, 8, 1): + shodan_version = parse_version(utils.package_version("shodan")) + if 'https_proxy' in parameters and shodan_version <= parse_version("1.8.1"): messages.append(["error", "Library 'shodan' needs to be updated. At least version 1.8.1 is required for HTTPS Proxy support."]) return messages diff --git a/intelmq/bots/collectors/shodan/collector_stream.py b/intelmq/bots/collectors/shodan/collector_stream.py index 2e8b9562bd..b37e2c2daf 100644 --- a/intelmq/bots/collectors/shodan/collector_stream.py +++ b/intelmq/bots/collectors/shodan/collector_stream.py @@ -11,14 +11,14 @@ * countries: A list of strings or a comma separated list with country codes * alert: An alert ID from monitor.shodan.io """ -import pkg_resources from http.client import IncompleteRead from urllib3.exceptions import ProtocolError, ReadTimeoutError +from packaging.version import parse as parse_version from requests.exceptions import ChunkedEncodingError, ConnectionError from typing import List, Optional -from intelmq.lib.bot import CollectorBot +from intelmq.lib.bot import CollectorBot, utils try: import shodan @@ -38,7 +38,7 @@ def init(self): raise ValueError("Library 'shodan' is needed but not installed.") self.set_request_parameters() - if tuple(int(v) for v in pkg_resources.get_distribution("shodan").version.split('.')) <= (1, 8, 1): + if parse_version(utils.package_version("shodan")) <= parse_version("1.8.1"): if self.proxy: raise ValueError('Proxies are given but shodan-python > 1.8.1 is needed for proxy support.') else: diff --git a/intelmq/bots/parsers/ioc_extractor/parser.py b/intelmq/bots/parsers/ioc_extractor/parser.py index 53a9eec07d..1f061e0ac3 100644 --- a/intelmq/bots/parsers/ioc_extractor/parser.py +++ b/intelmq/bots/parsers/ioc_extractor/parser.py @@ -21,7 +21,6 @@ classification_type : string with a valid classificationtype """ import re -import pkg_resources from typing import Optional from collections.abc import Iterable @@ -30,6 +29,7 @@ from intelmq.lib.exceptions import InvalidArgument from intelmq.lib.harmonization import ClassificationType from intelmq.lib.exceptions import MissingDependencyError +from packaging.version import parse as parse_version try: from url_normalize import url_normalize @@ -57,8 +57,8 @@ class IocExtractorParserBot(ParserBot): def init(self): if url_normalize is None: raise MissingDependencyError("url-normalize") - url_version = pkg_resources.get_distribution("url-normalize").version - if tuple(int(v) for v in url_version.split('.')) < (1, 4, 1) and self.default_scheme is not None: + url_version = utils.package_version("url-normalize") + if parse_version(url_version) < parse_version("1.4.1") and self.default_scheme is not None: raise ValueError("Parameter 'default_scheme' given but 'url-normalize' version %r does not support it. " "Get at least version '1.4.1'." % url_version) if get_tld is None: diff --git a/intelmq/lib/bot.py b/intelmq/lib/bot.py index 7c62502112..5eea765419 100644 --- a/intelmq/lib/bot.py +++ b/intelmq/lib/bot.py @@ -30,7 +30,6 @@ from copy import deepcopy from datetime import datetime, timedelta from typing import Any, List, Optional, Union, Tuple -from pkg_resources import resource_filename import intelmq.lib.message as libmessage from intelmq import (DEFAULT_LOGGING_PATH, @@ -929,7 +928,9 @@ def __load_harmonization_configuration(self): if self._standalone: raise else: - self._harmonization = utils.load_configuration(resource_filename('intelmq', 'etc/harmonization.conf')) + self._harmonization = utils.load_configuration( + utils.package_resource_path('intelmq', 'etc/harmonization.conf') + ) def new_event(self, *args, **kwargs): return libmessage.Event(*args, harmonization=self.harmonization, **kwargs) diff --git a/intelmq/lib/message.py b/intelmq/lib/message.py index eeff508722..7fa7191b54 100644 --- a/intelmq/lib/message.py +++ b/intelmq/lib/message.py @@ -15,7 +15,6 @@ from collections import defaultdict from typing import Any, Dict, Optional, Union, Tuple from collections.abc import Iterable, Sequence -from pkg_resources import resource_filename import intelmq.lib.exceptions as exceptions import intelmq.lib.harmonization @@ -113,7 +112,9 @@ def __init__(self, message: Union[dict, tuple] = (), auto: bool = False, harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE) except ValueError: # Fallback to internal harmonization file - harmonization = utils.load_configuration(resource_filename('intelmq', 'etc/harmonization.conf')) + harmonization = utils.load_configuration( + utils.package_resource_path('intelmq', 'etc/harmonization.conf') + ) try: self.harmonization_config = harmonization[classname] except KeyError: diff --git a/intelmq/lib/test.py b/intelmq/lib/test.py index ae55f63d2d..f640234d05 100644 --- a/intelmq/lib/test.py +++ b/intelmq/lib/test.py @@ -18,7 +18,6 @@ import unittest.mock as mock from itertools import chain -import pkg_resources import redis import intelmq.lib.message as message @@ -66,8 +65,7 @@ def mocked(conf_file): }} elif conf_file.startswith(CONFIG_DIR): confname = os.path.join('etc/', os.path.split(conf_file)[-1]) - fname = pkg_resources.resource_filename('intelmq', - confname) + fname = utils.package_resource_path('intelmq', confname) with open(fname) as fpconfig: return json.load(fpconfig) else: @@ -187,8 +185,9 @@ def setUpClass(cls): elif cls.use_cache and os.environ.get('INTELMQ_SKIP_REDIS'): cls.skipTest(cls, 'Requested cache requires deactivated Redis.') - harmonization = utils.load_configuration(pkg_resources.resource_filename('intelmq', - 'etc/harmonization.conf')) + harmonization = utils.load_configuration( + utils.package_resource_path('intelmq', 'etc/harmonization.conf') + ) def new_report(self, auto=False, examples=False): return message.Report(harmonization=self.harmonization, auto=auto) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index f5d4e66908..3ace707a0e 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -4,11 +4,10 @@ SPDX-License-Identifier: AGPL-3.0-or-later """ from collections import OrderedDict -from pkg_resources import resource_filename from pathlib import Path from intelmq import CONFIG_DIR -from intelmq.lib.utils import load_configuration, write_configuration +from intelmq.lib.utils import load_configuration, package_resource_path, write_configuration __all__ = ['v100_dev7_modify_syntax', 'v110_shadowserver_feednames', @@ -380,8 +379,8 @@ def harmonization(configuration, harmonization, dry_run, **kwargs): Checks if all harmonization fields and types are correct """ changed = None - original = load_configuration(resource_filename('intelmq', - 'etc/harmonization.conf')) + original = load_configuration(package_resource_path('intelmq', + 'etc/harmonization.conf')) for msg_type, msg in original.items(): if msg_type not in harmonization: harmonization[msg_type] = msg @@ -1008,7 +1007,7 @@ def v350_new_fields(configuration, harmonization, dry_run, **kwargs): return changed, configuration, harmonization builtin_harmonisation = load_configuration( - resource_filename("intelmq", "etc/harmonization.conf") + package_resource_path("intelmq", "etc/harmonization.conf") ) for field in [ "severity", diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index 0a4922d703..abc2570eb1 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -20,6 +20,7 @@ import grp import gzip import importlib +from importlib import resources import inspect import io import json @@ -52,9 +53,9 @@ from intelmq.lib.exceptions import DecodingError try: - from importlib.metadata import entry_points + from importlib.metadata import PackageNotFoundError, entry_points, version as package_version except ImportError: - from importlib_metadata import entry_points + from importlib_metadata import PackageNotFoundError, entry_points, version as package_version __all__ = ['base64_decode', 'base64_encode', 'decode', 'encode', @@ -62,7 +63,8 @@ 'reverse_readline', 'error_message_from_exc', 'parse_relative', 'RewindableFileHandle', 'file_name_from_response', - 'list_all_bots', 'get_global_settings', + 'list_all_bots', 'get_global_settings', 'package_resource_path', + 'package_version', ] # Used loglines format @@ -230,6 +232,17 @@ def load_configuration(configuration_filepath: str) -> dict: return config +def package_resource_path(package: str, resource: str = '') -> str: + """ + Return a filesystem path for a resource bundled in an installed package. + """ + resource_path = resources.files(package) + for part in resource.split('/'): + if part: + resource_path = resource_path.joinpath(part) + return str(resource_path) + + def write_configuration(configuration_filepath: str, content: dict, backup: bool = True, new=False, useyaml=True) -> Optional[bool]: diff --git a/intelmq/tests/bin/test_intelmqctl.py b/intelmq/tests/bin/test_intelmqctl.py index 2dc75cc671..d6d4c79444 100644 --- a/intelmq/tests/bin/test_intelmqctl.py +++ b/intelmq/tests/bin/test_intelmqctl.py @@ -9,8 +9,6 @@ from unittest import mock from pathlib import Path -from pkg_resources import resource_filename - import intelmq.bin.intelmqctl as ctl import intelmq.lib.utils as utils from intelmq.lib.test import skip_installation @@ -90,13 +88,15 @@ def _extend_config(self, path: str, to_extend: dict, useyaml: bool = True): utils.write_configuration(path, config, backup=False, useyaml=useyaml) def _load_default_harmonization(self): - default = utils.load_configuration(resource_filename('intelmq', - 'etc/harmonization.conf')) + default = utils.load_configuration( + utils.package_resource_path('intelmq', 'etc/harmonization.conf') + ) self._extend_config(self.tmp_harmonization, default, useyaml=False) def _load_default_runtime(self): - default = utils.load_configuration(resource_filename('intelmq', - 'etc/runtime.yaml')) + default = utils.load_configuration( + utils.package_resource_path('intelmq', 'etc/runtime.yaml') + ) self._extend_config(self.tmp_runtime, default) def tearDown(self): diff --git a/intelmq/tests/bin/test_psql_initdb.py b/intelmq/tests/bin/test_psql_initdb.py index bedb0ce6f4..f0c7a833a0 100644 --- a/intelmq/tests/bin/test_psql_initdb.py +++ b/intelmq/tests/bin/test_psql_initdb.py @@ -14,9 +14,8 @@ import tempfile import unittest -import pkg_resources - import intelmq.bin.intelmq_psql_initdb as psql_initdb +from intelmq.lib.utils import package_resource_path class TestPsqlInit(unittest.TestCase): @@ -55,7 +54,7 @@ def test_output(self): with open(os.path.join(os.path.dirname(__file__), 'initdb.sql')) as handle: expected = handle.read() - fname = pkg_resources.resource_filename('intelmq', 'etc/harmonization.conf') + fname = package_resource_path('intelmq', 'etc/harmonization.conf') self.assertEqual(psql_initdb.generate(fname).strip(), expected.strip()) def test_skip_generating_events_table_schema(self): diff --git a/intelmq/tests/bots/experts/asn_lookup/test_expert.py b/intelmq/tests/bots/experts/asn_lookup/test_expert.py index 788f9269e4..fa2a771870 100644 --- a/intelmq/tests/bots/experts/asn_lookup/test_expert.py +++ b/intelmq/tests/bots/experts/asn_lookup/test_expert.py @@ -9,12 +9,11 @@ import unittest -import pkg_resources - import intelmq.lib.test as test +from intelmq.lib.utils import package_resource_path from intelmq.bots.experts.asn_lookup.expert import ASNLookupExpertBot -ASN_DB = pkg_resources.resource_filename('intelmq', 'tests/bots/experts/asn_lookup/ipasn.dat') +ASN_DB = package_resource_path('intelmq', 'tests/bots/experts/asn_lookup/ipasn.dat') EXAMPLE_INPUT = {"__type": "Event", "source.ip": "93.184.216.34", # example.com "destination.ip": "192.0.43.8", # iana.org diff --git a/intelmq/tests/bots/experts/fake/test_expert.py b/intelmq/tests/bots/experts/fake/test_expert.py index 577c3a6c24..913bed2530 100644 --- a/intelmq/tests/bots/experts/fake/test_expert.py +++ b/intelmq/tests/bots/experts/fake/test_expert.py @@ -6,13 +6,12 @@ from json import loads as json_loads from ipaddress import ip_network, ip_address -import pkg_resources - import intelmq.lib.test as test +from intelmq.lib.utils import package_resource_path from intelmq.bots.experts.fake.expert import FakeExpertBot -FAKE_DB = pkg_resources.resource_filename('intelmq', 'tests/bots/experts/fake/data.json') -SEVERITY_DB = pkg_resources.resource_filename('intelmq', 'tests/bots/experts/fake/severity.json') +FAKE_DB = package_resource_path('intelmq', 'tests/bots/experts/fake/data.json') +SEVERITY_DB = package_resource_path('intelmq', 'tests/bots/experts/fake/severity.json') EXAMPLE_INPUT = {"__type": "Event", "source.ip": "93.184.216.34", # example.com } diff --git a/intelmq/tests/bots/experts/jinja/test_expert.py b/intelmq/tests/bots/experts/jinja/test_expert.py index 3b7cc13c46..6580ef26e9 100644 --- a/intelmq/tests/bots/experts/jinja/test_expert.py +++ b/intelmq/tests/bots/experts/jinja/test_expert.py @@ -8,8 +8,6 @@ import unittest import os -import pkg_resources - import intelmq.lib.test as test from intelmq.bots.experts.jinja.expert import JinjaExpertBot diff --git a/intelmq/tests/bots/experts/modify/test_expert.py b/intelmq/tests/bots/experts/modify/test_expert.py index 5c9111f82d..7517b33d4a 100644 --- a/intelmq/tests/bots/experts/modify/test_expert.py +++ b/intelmq/tests/bots/experts/modify/test_expert.py @@ -9,10 +9,8 @@ import unittest -from pkg_resources import resource_filename - import intelmq.lib.test as test -from intelmq.lib.utils import load_configuration +from intelmq.lib.utils import load_configuration, package_resource_path from intelmq.bots.experts.modify.expert import ModifyExpertBot EVENT_TEMPL = {"__type": "Event", @@ -78,8 +76,8 @@ class TestModifyExpertBot(test.BotTestCase, unittest.TestCase): @classmethod def set_bot(cls): cls.bot_reference = ModifyExpertBot - config_path = resource_filename('intelmq', - 'bots/experts/modify/examples/default.conf') + config_path = package_resource_path('intelmq', + 'bots/experts/modify/examples/default.conf') cls.sysconfig = {'configuration_path': config_path } @@ -96,8 +94,8 @@ def test_types(self): """ boolean, int etc """ - config_path = resource_filename('intelmq', - 'tests/bots/experts/modify/types.conf') + config_path = package_resource_path('intelmq', + 'tests/bots/experts/modify/types.conf') parameters = {'configuration_path': config_path, 'overwrite': True} self.input_message = INPUT[8:14] @@ -110,8 +108,8 @@ def test_overwrite(self): """ test if bot overwrites by default """ - config_path = resource_filename('intelmq', - 'tests/bots/experts/modify/overwrite.conf') + config_path = package_resource_path('intelmq', + 'tests/bots/experts/modify/overwrite.conf') self.input_message = INPUT[7] self.allowed_warning_count = 1 self.run_bot(parameters={'configuration_path': config_path}) @@ -121,8 +119,8 @@ def test_overwrite_not(self): """ test if bot does not overwrites if parameter is set """ - config_path = resource_filename('intelmq', - 'tests/bots/experts/modify/overwrite.conf') + config_path = package_resource_path('intelmq', + 'tests/bots/experts/modify/overwrite.conf') self.input_message = EVENT_TEMPL self.run_bot(parameters={'configuration_path': config_path, 'overwrite': False}) diff --git a/intelmq/tests/bots/experts/recordedfuture_iprisk/test_expert.py b/intelmq/tests/bots/experts/recordedfuture_iprisk/test_expert.py index d1baa5d9ff..4acac89979 100644 --- a/intelmq/tests/bots/experts/recordedfuture_iprisk/test_expert.py +++ b/intelmq/tests/bots/experts/recordedfuture_iprisk/test_expert.py @@ -7,11 +7,11 @@ Testing RF Risk node lookup """ import unittest -import pkg_resources import intelmq.lib.test as test +from intelmq.lib.utils import package_resource_path from intelmq.bots.experts.recordedfuture_iprisk.expert import RecordedFutureIPRiskExpertBot -RFR_DB = pkg_resources.resource_filename('intelmq', 'tests/bots/experts/recordedfuture_iprisk/iprisk.dat') +RFR_DB = package_resource_path('intelmq', 'tests/bots/experts/recordedfuture_iprisk/iprisk.dat') EXAMPLE_INPUT = {"__type": "Event", "source.ip": "192.168.0.1", "destination.ip": "192.0.43.8", diff --git a/intelmq/tests/bots/experts/tor_nodes/test_expert.py b/intelmq/tests/bots/experts/tor_nodes/test_expert.py index a3e77d47fc..74fdb7a6a5 100644 --- a/intelmq/tests/bots/experts/tor_nodes/test_expert.py +++ b/intelmq/tests/bots/experts/tor_nodes/test_expert.py @@ -9,12 +9,11 @@ import unittest -import pkg_resources - import intelmq.lib.test as test +from intelmq.lib.utils import package_resource_path from intelmq.bots.experts.tor_nodes.expert import TorExpertBot -TOR_DB = pkg_resources.resource_filename('intelmq', 'tests/bots/experts/tor_nodes/tor_nodes.dat') +TOR_DB = package_resource_path('intelmq', 'tests/bots/experts/tor_nodes/tor_nodes.dat') EXAMPLE_INPUT = {"__type": "Event", "source.ip": "192.168.0.1", "destination.ip": "192.0.43.8", diff --git a/intelmq/tests/bots/parsers/hibp/test_parser_callback.py b/intelmq/tests/bots/parsers/hibp/test_parser_callback.py index d8264b5083..1b139a13f4 100644 --- a/intelmq/tests/bots/parsers/hibp/test_parser_callback.py +++ b/intelmq/tests/bots/parsers/hibp/test_parser_callback.py @@ -8,13 +8,12 @@ """ import json import unittest -import pkg_resources import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.hibp.parser_callback import HIBPCallbackParserBot -BREACHREQUEST = json.load(open(pkg_resources.resource_filename('intelmq', 'tests/bots/parsers/hibp/breach_callbacktest.json'))) +BREACHREQUEST = json.load(open(utils.package_resource_path('intelmq', 'tests/bots/parsers/hibp/breach_callbacktest.json'))) BREACHRAW = utils.base64_encode(json.dumps(BREACHREQUEST, sort_keys=True)) BR_REP = {"feed.name": "HIBP Enterprise", "time.observation": "2019-03-01T01:01:01+00:00", @@ -34,7 +33,7 @@ "__type": "Event" } -PASTEREQUEST = json.load(open(pkg_resources.resource_filename('intelmq', 'tests/bots/parsers/hibp/paste_callbacktest.json'))) +PASTEREQUEST = json.load(open(utils.package_resource_path('intelmq', 'tests/bots/parsers/hibp/paste_callbacktest.json'))) PASTERAW = utils.base64_encode(json.dumps(PASTEREQUEST, sort_keys=True)) PA_REP = {"feed.name": "HIBP Enterprise", "time.observation": "2019-03-01T01:01:01+00:00", @@ -55,7 +54,7 @@ } -BREACHREALREQUEST = json.load(open(pkg_resources.resource_filename('intelmq', 'tests/bots/parsers/hibp/breach_real.json'))) +BREACHREALREQUEST = json.load(open(utils.package_resource_path('intelmq', 'tests/bots/parsers/hibp/breach_real.json'))) BREACHREALRAW = utils.base64_encode(json.dumps(BREACHREALREQUEST, sort_keys=True)) BR_REAL_REP = {"feed.name": "HIBP Enterprise", "time.observation": "2019-03-01T01:01:01+00:00", diff --git a/intelmq/tests/lib/test_message.py b/intelmq/tests/lib/test_message.py index ac1e1cbcdd..f1269a9bb7 100644 --- a/intelmq/tests/lib/test_message.py +++ b/intelmq/tests/lib/test_message.py @@ -13,14 +13,12 @@ import json import unittest -import pkg_resources - import intelmq.lib.exceptions as exceptions import intelmq.lib.message as message # noqa -from intelmq.lib.utils import load_configuration +from intelmq.lib.utils import load_configuration, package_resource_path -HARM = load_configuration(pkg_resources.resource_filename('intelmq', - 'etc/harmonization.conf')) +HARM = load_configuration(package_resource_path('intelmq', + 'etc/harmonization.conf')) LOREM_BASE64 = 'bG9yZW0gaXBzdW0=' DOLOR_BASE64 = 'ZG9sb3Igc2l0IGFtZXQ=' diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index 72dbb1f592..19282810c2 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -6,11 +6,10 @@ Tests the upgrade functions. """ import unittest -import pkg_resources from copy import deepcopy import intelmq.lib.upgrades as upgrades -from intelmq.lib.utils import load_configuration +from intelmq.lib.utils import load_configuration, package_resource_path V202 = {"global": {}, @@ -261,8 +260,8 @@ "module": "intelmq.bots.collectors.misp.collector", "parameters": { "http_verify_cert": False}}} -HARM = load_configuration(pkg_resources.resource_filename('intelmq', - 'etc/harmonization.conf')) +HARM = load_configuration(package_resource_path('intelmq', + 'etc/harmonization.conf')) V210_HARM = deepcopy(HARM) del V210_HARM['report']['extra'] MISSING_REPORT = deepcopy(HARM) diff --git a/intelmq/tests/lib/test_utils.py b/intelmq/tests/lib/test_utils.py index ddb34408a3..c9b63b86b3 100644 --- a/intelmq/tests/lib/test_utils.py +++ b/intelmq/tests/lib/test_utils.py @@ -22,7 +22,6 @@ import cerberus import dns.resolver -import pkg_resources import requests import termstyle from ruamel.yaml.scanner import ScannerError @@ -51,7 +50,7 @@ def new_get_runtime() -> dict: - runtime_conf = utils.load_configuration(pkg_resources.resource_filename('intelmq', 'etc/runtime.yaml')) + runtime_conf = utils.load_configuration(utils.package_resource_path('intelmq', 'etc/runtime.yaml')) if 'global' not in runtime_conf: runtime_conf['global'] = {} runtime_conf['global']['http_proxy'] = 'http://localhost:8080' diff --git a/intelmq/tests/test_conf.py b/intelmq/tests/test_conf.py index 76e09a63e8..60bbdc06af 100644 --- a/intelmq/tests/test_conf.py +++ b/intelmq/tests/test_conf.py @@ -17,14 +17,13 @@ import unittest import cerberus -import pkg_resources from ruamel.yaml import YAML import intelmq.bots import intelmq.lib.harmonization as harmonization -from intelmq.lib.utils import lazy_int +from intelmq.lib.utils import lazy_int, package_resource_path yaml = YAML(typ="safe", pure=True) yaml.default_flow_style = False # This makes the configuration more readable, diff-friendly and consistent. @@ -46,8 +45,8 @@ def to_unsorted_json(obj): separators=(',', ': ')) + '\n' -CONF_FILES = {'harmonization': pkg_resources.resource_filename('intelmq', 'etc/harmonization.conf'), - 'runtime': pkg_resources.resource_filename('intelmq', 'etc/runtime.yaml')} +CONF_FILES = {'harmonization': package_resource_path('intelmq', 'etc/harmonization.conf'), + 'runtime': package_resource_path('intelmq', 'etc/runtime.yaml')} class TestConf(unittest.TestCase): @@ -113,8 +112,7 @@ def convert_cerberus_schema(self, schema: str) -> str: def test_feeds(self): with open(os.path.join(os.path.dirname(__file__), 'assets/feeds.schema.json')) as handle: schema = json.loads(self.convert_cerberus_schema(handle.read())) - with open(pkg_resources.resource_filename('intelmq', - 'etc/feeds.yaml'), encoding='UTF-8') as handle: + with open(package_resource_path('intelmq', 'etc/feeds.yaml'), encoding='UTF-8') as handle: feeds = yaml.load(handle) v = cerberus.Validator(schema) From 02cb7cb55cb26ccf95b5a9283ebdbb7e0d5efa1b Mon Sep 17 00:00:00 2001 From: Mirochill <200482516+Mirochill@users.noreply.github.com> Date: Sat, 23 May 2026 09:28:40 +0200 Subject: [PATCH 2/2] Fix resource lookup under import mocks --- intelmq/lib/upgrades.py | 5 +++-- intelmq/lib/utils.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 3ace707a0e..32005cd4bd 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -379,8 +379,9 @@ def harmonization(configuration, harmonization, dry_run, **kwargs): Checks if all harmonization fields and types are correct """ changed = None - original = load_configuration(package_resource_path('intelmq', - 'etc/harmonization.conf')) + original = load_configuration( + package_resource_path('intelmq', 'etc/harmonization.conf') + ) for msg_type, msg in original.items(): if msg_type not in harmonization: harmonization[msg_type] = msg diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index abc2570eb1..80cff45bcf 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -236,7 +236,8 @@ def package_resource_path(package: str, resource: str = '') -> str: """ Return a filesystem path for a resource bundled in an installed package. """ - resource_path = resources.files(package) + package_reference = sys.modules.get(package, package) + resource_path = resources.files(package_reference) for part in resource.split('/'): if part: resource_path = resource_path.joinpath(part)