diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index a7c1ebc29a..d2ba6dae4f 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -52,10 +52,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install setuptools for pkg_resources for Python 3.12+, #2569 - if: ${{ matrix.python-version >= '3.12' }} - run: pip install 'setuptools<82' - - name: Install dependencies for full run if: ${{ matrix.type == 'full' }} env: diff --git a/intelmq/bin/intelmqctl.py b/intelmq/bin/intelmqctl.py index 9668e0327c..0f4e9f4390 100644 --- a/intelmq/bin/intelmqctl.py +++ b/intelmq/bin/intelmqctl.py @@ -17,7 +17,6 @@ import traceback import time -import pkg_resources from ruamel.yaml import YAML from intelmq import (DEFAULT_LOGGING_LEVEL, # noqa: F401 @@ -127,8 +126,8 @@ def __init__(self, interactive: bool = False, returntype: ReturnType = ReturnTyp APPNAME = "intelmqctl" try: - VERSION = pkg_resources.get_distribution("intelmq").version - except pkg_resources.DistributionNotFound: # pragma: no cover + VERSION = utils.package_version("intelmq") + except utils.PackageNotFoundError: # pragma: no cover # can only happen in interactive mode self._logger.error('No valid IntelMQ installation found: DistributionNotFound') sys.exit(1) diff --git a/intelmq/bin/intelmqsetup.py b/intelmq/bin/intelmqsetup.py index 5d4f65d7c0..5b7a1d61c4 100755 --- a/intelmq/bin/intelmqsetup.py +++ b/intelmq/bin/intelmqsetup.py @@ -22,7 +22,6 @@ import shutil import stat import sys -import pkg_resources from grp import getgrnam from pathlib import Path @@ -53,6 +52,7 @@ from intelmq import (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH, VAR_STATE_PATH, STATE_FILE_PATH) from intelmq.bin.intelmqctl import IntelMQController +from intelmq.lib.utils import package_resource_path, package_version FILE_OUTPUT_PATH = Path(VAR_STATE_PATH) / 'file-output/' @@ -157,7 +157,7 @@ def intelmqsetup_core(ownership=True, state_file=STATE_FILE_PATH): create_directory(DEFAULT_LOGGING_PATH, 0o40755) create_directory(CONFIG_DIR, 0o40775) - example_path = Path(pkg_resources.resource_filename('intelmq', 'etc')) + example_path = Path(package_resource_path('intelmq', 'etc')) example_confs = [example_path / 'runtime.yaml', example_path / 'harmonization.conf'] for example_conf in example_confs: fname = Path(example_conf).name @@ -200,7 +200,7 @@ def intelmqsetup_api(ownership: bool = True, webserver_user: Optional[str] = Non if ownership: change_owner(ETC_INTELMQ_MANAGER, group='intelmq') - base = Path(pkg_resources.resource_filename('intelmq_api', '')).parent + base = Path(package_resource_path('intelmq_api')).parent api_config = base / 'etc/intelmq/api-config.json' etc_intelmq_config = ETC_INTELMQ / 'api-config.json' api_sudoers = base / 'etc/intelmq/api-sudoers.conf' @@ -249,7 +249,7 @@ def intelmqsetup_api(ownership: bool = True, webserver_user: Optional[str] = Non def intelmqsetup_api_webserver_configuration(webserver_configuration_directory: Optional[str] = None): webserver_configuration_dir = webserver_configuration_directory or find_webserver_configuration_directory() - api_config = Path(pkg_resources.resource_filename('intelmq_api', '')).parent / 'etc/intelmq/api-apache.conf' + api_config = Path(package_resource_path('intelmq_api')).parent / 'etc/intelmq/api-apache.conf' apache_api_config = webserver_configuration_dir / 'api-apache.conf' if api_config.exists() and not apache_api_config.exists(): shutil.copy(api_config, apache_api_config) @@ -266,9 +266,9 @@ def intelmqsetup_api_webserver_configuration(webserver_configuration_directory: def intelmqsetup_manager_webserver_configuration(webserver_configuration_directory: Optional[str] = None): webserver_configuration_dir = webserver_configuration_directory or find_webserver_configuration_directory() - manager_config_1 = Path(pkg_resources.resource_filename('intelmq_manager', '')).parent / 'etc/intelmq/manager-apache.conf' + manager_config_1 = Path(package_resource_path('intelmq_manager')).parent / 'etc/intelmq/manager-apache.conf' # IntelMQ Manager >= 3.1.0 - manager_config_2 = Path(pkg_resources.resource_filename('intelmq_manager', '')) / 'manager-apache.conf' + manager_config_2 = Path(package_resource_path('intelmq_manager')) / 'manager-apache.conf' manager_config = manager_config_2 if manager_config_2.exists() else manager_config_1 apache_manager_config = webserver_configuration_dir / 'manager-apache.conf' if manager_config.exists() and not apache_manager_config.exists(): @@ -287,7 +287,7 @@ def intelmqsetup_manager_generate(): print('Unable to build intelmq-manager files. Installed version of intelmq-manager is too old, at least version 3.1.0 is required.', file=sys.stderr) return - src_dir = Path(pkg_resources.resource_filename('intelmq_manager', '')) + src_dir = Path(package_resource_path('intelmq_manager')) html_dir_destination = Path('/usr/share/intelmq_manager/html') if not src_dir.as_posix().startswith('/usr/'): @@ -338,7 +338,7 @@ def main(): else: print('Skipping intelmq-manager configuration.') if intelmq_manager and not args.skip_manager: - manager_version = pkg_resources.get_distribution('intelmq-manager').version + manager_version = package_version('intelmq-manager') print(f'Generate and save intelmq-manager (version {manager_version}) static files.') intelmqsetup_manager_generate() diff --git a/intelmq/bots/collectors/shodan/collector_alert.py b/intelmq/bots/collectors/shodan/collector_alert.py index 7b7fde26ea..d0cce66a2e 100644 --- a/intelmq/bots/collectors/shodan/collector_alert.py +++ b/intelmq/bots/collectors/shodan/collector_alert.py @@ -3,11 +3,12 @@ SPDX-License-Identifier: AGPL-3.0-or-later """ -from pkg_resources import get_distribution from json import dumps as json_dumps from typing import Optional -from intelmq.lib.bot import CollectorBot +from packaging.version import parse as parse_version + +from intelmq.lib.bot import CollectorBot, utils try: from shodan import Shodan @@ -32,7 +33,7 @@ def init(self): 'https': self.https_proxy} if self.https_proxy else {}) - if tuple(int(v) for v in get_distribution("shodan").version.split('.')) <= (1, 8, 1): + if parse_version(utils.package_version("shodan")) <= parse_version("1.8.1"): if self.proxy: raise ValueError('Proxies are given but shodan-python > 1.8.1 is needed for proxy support.') else: @@ -59,8 +60,8 @@ def check(parameters: dict) -> Optional[list[list[str]]]: else: messages.append(["error", "Library 'shodan' is needed but not installed."]) else: - shodan_version = tuple(int(v) for v in get_distribution("shodan").version.split('.')) - if 'https_proxy' in parameters and shodan_version <= (1, 8, 1): + shodan_version = parse_version(utils.package_version("shodan")) + if 'https_proxy' in parameters and shodan_version <= parse_version("1.8.1"): messages.append(["error", "Library 'shodan' needs to be updated. At least version 1.8.1 is required for HTTPS Proxy support."]) return messages diff --git a/intelmq/bots/collectors/shodan/collector_stream.py b/intelmq/bots/collectors/shodan/collector_stream.py index 2e8b9562bd..b37e2c2daf 100644 --- a/intelmq/bots/collectors/shodan/collector_stream.py +++ b/intelmq/bots/collectors/shodan/collector_stream.py @@ -11,14 +11,14 @@ * countries: A list of strings or a comma separated list with country codes * alert: An alert ID from monitor.shodan.io """ -import pkg_resources from http.client import IncompleteRead from urllib3.exceptions import ProtocolError, ReadTimeoutError +from packaging.version import parse as parse_version from requests.exceptions import ChunkedEncodingError, ConnectionError from typing import List, Optional -from intelmq.lib.bot import CollectorBot +from intelmq.lib.bot import CollectorBot, utils try: import shodan @@ -38,7 +38,7 @@ def init(self): raise ValueError("Library 'shodan' is needed but not installed.") self.set_request_parameters() - if tuple(int(v) for v in pkg_resources.get_distribution("shodan").version.split('.')) <= (1, 8, 1): + if parse_version(utils.package_version("shodan")) <= parse_version("1.8.1"): if self.proxy: raise ValueError('Proxies are given but shodan-python > 1.8.1 is needed for proxy support.') else: diff --git a/intelmq/bots/parsers/ioc_extractor/parser.py b/intelmq/bots/parsers/ioc_extractor/parser.py index 53a9eec07d..1f061e0ac3 100644 --- a/intelmq/bots/parsers/ioc_extractor/parser.py +++ b/intelmq/bots/parsers/ioc_extractor/parser.py @@ -21,7 +21,6 @@ classification_type : string with a valid classificationtype """ import re -import pkg_resources from typing import Optional from collections.abc import Iterable @@ -30,6 +29,7 @@ from intelmq.lib.exceptions import InvalidArgument from intelmq.lib.harmonization import ClassificationType from intelmq.lib.exceptions import MissingDependencyError +from packaging.version import parse as parse_version try: from url_normalize import url_normalize @@ -57,8 +57,8 @@ class IocExtractorParserBot(ParserBot): def init(self): if url_normalize is None: raise MissingDependencyError("url-normalize") - url_version = pkg_resources.get_distribution("url-normalize").version - if tuple(int(v) for v in url_version.split('.')) < (1, 4, 1) and self.default_scheme is not None: + url_version = utils.package_version("url-normalize") + if parse_version(url_version) < parse_version("1.4.1") and self.default_scheme is not None: raise ValueError("Parameter 'default_scheme' given but 'url-normalize' version %r does not support it. " "Get at least version '1.4.1'." % url_version) if get_tld is None: diff --git a/intelmq/lib/bot.py b/intelmq/lib/bot.py index 7c62502112..5eea765419 100644 --- a/intelmq/lib/bot.py +++ b/intelmq/lib/bot.py @@ -30,7 +30,6 @@ from copy import deepcopy from datetime import datetime, timedelta from typing import Any, List, Optional, Union, Tuple -from pkg_resources import resource_filename import intelmq.lib.message as libmessage from intelmq import (DEFAULT_LOGGING_PATH, @@ -929,7 +928,9 @@ def __load_harmonization_configuration(self): if self._standalone: raise else: - self._harmonization = utils.load_configuration(resource_filename('intelmq', 'etc/harmonization.conf')) + self._harmonization = utils.load_configuration( + utils.package_resource_path('intelmq', 'etc/harmonization.conf') + ) def new_event(self, *args, **kwargs): return libmessage.Event(*args, harmonization=self.harmonization, **kwargs) diff --git a/intelmq/lib/message.py b/intelmq/lib/message.py index eeff508722..7fa7191b54 100644 --- a/intelmq/lib/message.py +++ b/intelmq/lib/message.py @@ -15,7 +15,6 @@ from collections import defaultdict from typing import Any, Dict, Optional, Union, Tuple from collections.abc import Iterable, Sequence -from pkg_resources import resource_filename import intelmq.lib.exceptions as exceptions import intelmq.lib.harmonization @@ -113,7 +112,9 @@ def __init__(self, message: Union[dict, tuple] = (), auto: bool = False, harmonization = utils.load_configuration(HARMONIZATION_CONF_FILE) except ValueError: # Fallback to internal harmonization file - harmonization = utils.load_configuration(resource_filename('intelmq', 'etc/harmonization.conf')) + harmonization = utils.load_configuration( + utils.package_resource_path('intelmq', 'etc/harmonization.conf') + ) try: self.harmonization_config = harmonization[classname] except KeyError: diff --git a/intelmq/lib/test.py b/intelmq/lib/test.py index ae55f63d2d..f640234d05 100644 --- a/intelmq/lib/test.py +++ b/intelmq/lib/test.py @@ -18,7 +18,6 @@ import unittest.mock as mock from itertools import chain -import pkg_resources import redis import intelmq.lib.message as message @@ -66,8 +65,7 @@ def mocked(conf_file): }} elif conf_file.startswith(CONFIG_DIR): confname = os.path.join('etc/', os.path.split(conf_file)[-1]) - fname = pkg_resources.resource_filename('intelmq', - confname) + fname = utils.package_resource_path('intelmq', confname) with open(fname) as fpconfig: return json.load(fpconfig) else: @@ -187,8 +185,9 @@ def setUpClass(cls): elif cls.use_cache and os.environ.get('INTELMQ_SKIP_REDIS'): cls.skipTest(cls, 'Requested cache requires deactivated Redis.') - harmonization = utils.load_configuration(pkg_resources.resource_filename('intelmq', - 'etc/harmonization.conf')) + harmonization = utils.load_configuration( + utils.package_resource_path('intelmq', 'etc/harmonization.conf') + ) def new_report(self, auto=False, examples=False): return message.Report(harmonization=self.harmonization, auto=auto) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index f5d4e66908..32005cd4bd 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -4,11 +4,10 @@ SPDX-License-Identifier: AGPL-3.0-or-later """ from collections import OrderedDict -from pkg_resources import resource_filename from pathlib import Path from intelmq import CONFIG_DIR -from intelmq.lib.utils import load_configuration, write_configuration +from intelmq.lib.utils import load_configuration, package_resource_path, write_configuration __all__ = ['v100_dev7_modify_syntax', 'v110_shadowserver_feednames', @@ -380,8 +379,9 @@ def harmonization(configuration, harmonization, dry_run, **kwargs): Checks if all harmonization fields and types are correct """ changed = None - original = load_configuration(resource_filename('intelmq', - 'etc/harmonization.conf')) + original = load_configuration( + package_resource_path('intelmq', 'etc/harmonization.conf') + ) for msg_type, msg in original.items(): if msg_type not in harmonization: harmonization[msg_type] = msg @@ -1008,7 +1008,7 @@ def v350_new_fields(configuration, harmonization, dry_run, **kwargs): return changed, configuration, harmonization builtin_harmonisation = load_configuration( - resource_filename("intelmq", "etc/harmonization.conf") + package_resource_path("intelmq", "etc/harmonization.conf") ) for field in [ "severity", diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index 0a4922d703..80cff45bcf 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -20,6 +20,7 @@ import grp import gzip import importlib +from importlib import resources import inspect import io import json @@ -52,9 +53,9 @@ from intelmq.lib.exceptions import DecodingError try: - from importlib.metadata import entry_points + from importlib.metadata import PackageNotFoundError, entry_points, version as package_version except ImportError: - from importlib_metadata import entry_points + from importlib_metadata import PackageNotFoundError, entry_points, version as package_version __all__ = ['base64_decode', 'base64_encode', 'decode', 'encode', @@ -62,7 +63,8 @@ 'reverse_readline', 'error_message_from_exc', 'parse_relative', 'RewindableFileHandle', 'file_name_from_response', - 'list_all_bots', 'get_global_settings', + 'list_all_bots', 'get_global_settings', 'package_resource_path', + 'package_version', ] # Used loglines format @@ -230,6 +232,18 @@ def load_configuration(configuration_filepath: str) -> dict: return config +def package_resource_path(package: str, resource: str = '') -> str: + """ + Return a filesystem path for a resource bundled in an installed package. + """ + package_reference = sys.modules.get(package, package) + resource_path = resources.files(package_reference) + for part in resource.split('/'): + if part: + resource_path = resource_path.joinpath(part) + return str(resource_path) + + def write_configuration(configuration_filepath: str, content: dict, backup: bool = True, new=False, useyaml=True) -> Optional[bool]: diff --git a/intelmq/tests/bin/test_intelmqctl.py b/intelmq/tests/bin/test_intelmqctl.py index 2dc75cc671..d6d4c79444 100644 --- a/intelmq/tests/bin/test_intelmqctl.py +++ b/intelmq/tests/bin/test_intelmqctl.py @@ -9,8 +9,6 @@ from unittest import mock from pathlib import Path -from pkg_resources import resource_filename - import intelmq.bin.intelmqctl as ctl import intelmq.lib.utils as utils from intelmq.lib.test import skip_installation @@ -90,13 +88,15 @@ def _extend_config(self, path: str, to_extend: dict, useyaml: bool = True): utils.write_configuration(path, config, backup=False, useyaml=useyaml) def _load_default_harmonization(self): - default = utils.load_configuration(resource_filename('intelmq', - 'etc/harmonization.conf')) + default = utils.load_configuration( + utils.package_resource_path('intelmq', 'etc/harmonization.conf') + ) self._extend_config(self.tmp_harmonization, default, useyaml=False) def _load_default_runtime(self): - default = utils.load_configuration(resource_filename('intelmq', - 'etc/runtime.yaml')) + default = utils.load_configuration( + utils.package_resource_path('intelmq', 'etc/runtime.yaml') + ) self._extend_config(self.tmp_runtime, default) def tearDown(self): diff --git a/intelmq/tests/bin/test_psql_initdb.py b/intelmq/tests/bin/test_psql_initdb.py index bedb0ce6f4..f0c7a833a0 100644 --- a/intelmq/tests/bin/test_psql_initdb.py +++ b/intelmq/tests/bin/test_psql_initdb.py @@ -14,9 +14,8 @@ import tempfile import unittest -import pkg_resources - import intelmq.bin.intelmq_psql_initdb as psql_initdb +from intelmq.lib.utils import package_resource_path class TestPsqlInit(unittest.TestCase): @@ -55,7 +54,7 @@ def test_output(self): with open(os.path.join(os.path.dirname(__file__), 'initdb.sql')) as handle: expected = handle.read() - fname = pkg_resources.resource_filename('intelmq', 'etc/harmonization.conf') + fname = package_resource_path('intelmq', 'etc/harmonization.conf') self.assertEqual(psql_initdb.generate(fname).strip(), expected.strip()) def test_skip_generating_events_table_schema(self): diff --git a/intelmq/tests/bots/experts/asn_lookup/test_expert.py b/intelmq/tests/bots/experts/asn_lookup/test_expert.py index 788f9269e4..fa2a771870 100644 --- a/intelmq/tests/bots/experts/asn_lookup/test_expert.py +++ b/intelmq/tests/bots/experts/asn_lookup/test_expert.py @@ -9,12 +9,11 @@ import unittest -import pkg_resources - import intelmq.lib.test as test +from intelmq.lib.utils import package_resource_path from intelmq.bots.experts.asn_lookup.expert import ASNLookupExpertBot -ASN_DB = pkg_resources.resource_filename('intelmq', 'tests/bots/experts/asn_lookup/ipasn.dat') +ASN_DB = package_resource_path('intelmq', 'tests/bots/experts/asn_lookup/ipasn.dat') EXAMPLE_INPUT = {"__type": "Event", "source.ip": "93.184.216.34", # example.com "destination.ip": "192.0.43.8", # iana.org diff --git a/intelmq/tests/bots/experts/fake/test_expert.py b/intelmq/tests/bots/experts/fake/test_expert.py index 577c3a6c24..913bed2530 100644 --- a/intelmq/tests/bots/experts/fake/test_expert.py +++ b/intelmq/tests/bots/experts/fake/test_expert.py @@ -6,13 +6,12 @@ from json import loads as json_loads from ipaddress import ip_network, ip_address -import pkg_resources - import intelmq.lib.test as test +from intelmq.lib.utils import package_resource_path from intelmq.bots.experts.fake.expert import FakeExpertBot -FAKE_DB = pkg_resources.resource_filename('intelmq', 'tests/bots/experts/fake/data.json') -SEVERITY_DB = pkg_resources.resource_filename('intelmq', 'tests/bots/experts/fake/severity.json') +FAKE_DB = package_resource_path('intelmq', 'tests/bots/experts/fake/data.json') +SEVERITY_DB = package_resource_path('intelmq', 'tests/bots/experts/fake/severity.json') EXAMPLE_INPUT = {"__type": "Event", "source.ip": "93.184.216.34", # example.com } diff --git a/intelmq/tests/bots/experts/jinja/test_expert.py b/intelmq/tests/bots/experts/jinja/test_expert.py index 3b7cc13c46..6580ef26e9 100644 --- a/intelmq/tests/bots/experts/jinja/test_expert.py +++ b/intelmq/tests/bots/experts/jinja/test_expert.py @@ -8,8 +8,6 @@ import unittest import os -import pkg_resources - import intelmq.lib.test as test from intelmq.bots.experts.jinja.expert import JinjaExpertBot diff --git a/intelmq/tests/bots/experts/modify/test_expert.py b/intelmq/tests/bots/experts/modify/test_expert.py index 5c9111f82d..7517b33d4a 100644 --- a/intelmq/tests/bots/experts/modify/test_expert.py +++ b/intelmq/tests/bots/experts/modify/test_expert.py @@ -9,10 +9,8 @@ import unittest -from pkg_resources import resource_filename - import intelmq.lib.test as test -from intelmq.lib.utils import load_configuration +from intelmq.lib.utils import load_configuration, package_resource_path from intelmq.bots.experts.modify.expert import ModifyExpertBot EVENT_TEMPL = {"__type": "Event", @@ -78,8 +76,8 @@ class TestModifyExpertBot(test.BotTestCase, unittest.TestCase): @classmethod def set_bot(cls): cls.bot_reference = ModifyExpertBot - config_path = resource_filename('intelmq', - 'bots/experts/modify/examples/default.conf') + config_path = package_resource_path('intelmq', + 'bots/experts/modify/examples/default.conf') cls.sysconfig = {'configuration_path': config_path } @@ -96,8 +94,8 @@ def test_types(self): """ boolean, int etc """ - config_path = resource_filename('intelmq', - 'tests/bots/experts/modify/types.conf') + config_path = package_resource_path('intelmq', + 'tests/bots/experts/modify/types.conf') parameters = {'configuration_path': config_path, 'overwrite': True} self.input_message = INPUT[8:14] @@ -110,8 +108,8 @@ def test_overwrite(self): """ test if bot overwrites by default """ - config_path = resource_filename('intelmq', - 'tests/bots/experts/modify/overwrite.conf') + config_path = package_resource_path('intelmq', + 'tests/bots/experts/modify/overwrite.conf') self.input_message = INPUT[7] self.allowed_warning_count = 1 self.run_bot(parameters={'configuration_path': config_path}) @@ -121,8 +119,8 @@ def test_overwrite_not(self): """ test if bot does not overwrites if parameter is set """ - config_path = resource_filename('intelmq', - 'tests/bots/experts/modify/overwrite.conf') + config_path = package_resource_path('intelmq', + 'tests/bots/experts/modify/overwrite.conf') self.input_message = EVENT_TEMPL self.run_bot(parameters={'configuration_path': config_path, 'overwrite': False}) diff --git a/intelmq/tests/bots/experts/recordedfuture_iprisk/test_expert.py b/intelmq/tests/bots/experts/recordedfuture_iprisk/test_expert.py index d1baa5d9ff..4acac89979 100644 --- a/intelmq/tests/bots/experts/recordedfuture_iprisk/test_expert.py +++ b/intelmq/tests/bots/experts/recordedfuture_iprisk/test_expert.py @@ -7,11 +7,11 @@ Testing RF Risk node lookup """ import unittest -import pkg_resources import intelmq.lib.test as test +from intelmq.lib.utils import package_resource_path from intelmq.bots.experts.recordedfuture_iprisk.expert import RecordedFutureIPRiskExpertBot -RFR_DB = pkg_resources.resource_filename('intelmq', 'tests/bots/experts/recordedfuture_iprisk/iprisk.dat') +RFR_DB = package_resource_path('intelmq', 'tests/bots/experts/recordedfuture_iprisk/iprisk.dat') EXAMPLE_INPUT = {"__type": "Event", "source.ip": "192.168.0.1", "destination.ip": "192.0.43.8", diff --git a/intelmq/tests/bots/experts/tor_nodes/test_expert.py b/intelmq/tests/bots/experts/tor_nodes/test_expert.py index a3e77d47fc..74fdb7a6a5 100644 --- a/intelmq/tests/bots/experts/tor_nodes/test_expert.py +++ b/intelmq/tests/bots/experts/tor_nodes/test_expert.py @@ -9,12 +9,11 @@ import unittest -import pkg_resources - import intelmq.lib.test as test +from intelmq.lib.utils import package_resource_path from intelmq.bots.experts.tor_nodes.expert import TorExpertBot -TOR_DB = pkg_resources.resource_filename('intelmq', 'tests/bots/experts/tor_nodes/tor_nodes.dat') +TOR_DB = package_resource_path('intelmq', 'tests/bots/experts/tor_nodes/tor_nodes.dat') EXAMPLE_INPUT = {"__type": "Event", "source.ip": "192.168.0.1", "destination.ip": "192.0.43.8", diff --git a/intelmq/tests/bots/parsers/hibp/test_parser_callback.py b/intelmq/tests/bots/parsers/hibp/test_parser_callback.py index d8264b5083..1b139a13f4 100644 --- a/intelmq/tests/bots/parsers/hibp/test_parser_callback.py +++ b/intelmq/tests/bots/parsers/hibp/test_parser_callback.py @@ -8,13 +8,12 @@ """ import json import unittest -import pkg_resources import intelmq.lib.test as test import intelmq.lib.utils as utils from intelmq.bots.parsers.hibp.parser_callback import HIBPCallbackParserBot -BREACHREQUEST = json.load(open(pkg_resources.resource_filename('intelmq', 'tests/bots/parsers/hibp/breach_callbacktest.json'))) +BREACHREQUEST = json.load(open(utils.package_resource_path('intelmq', 'tests/bots/parsers/hibp/breach_callbacktest.json'))) BREACHRAW = utils.base64_encode(json.dumps(BREACHREQUEST, sort_keys=True)) BR_REP = {"feed.name": "HIBP Enterprise", "time.observation": "2019-03-01T01:01:01+00:00", @@ -34,7 +33,7 @@ "__type": "Event" } -PASTEREQUEST = json.load(open(pkg_resources.resource_filename('intelmq', 'tests/bots/parsers/hibp/paste_callbacktest.json'))) +PASTEREQUEST = json.load(open(utils.package_resource_path('intelmq', 'tests/bots/parsers/hibp/paste_callbacktest.json'))) PASTERAW = utils.base64_encode(json.dumps(PASTEREQUEST, sort_keys=True)) PA_REP = {"feed.name": "HIBP Enterprise", "time.observation": "2019-03-01T01:01:01+00:00", @@ -55,7 +54,7 @@ } -BREACHREALREQUEST = json.load(open(pkg_resources.resource_filename('intelmq', 'tests/bots/parsers/hibp/breach_real.json'))) +BREACHREALREQUEST = json.load(open(utils.package_resource_path('intelmq', 'tests/bots/parsers/hibp/breach_real.json'))) BREACHREALRAW = utils.base64_encode(json.dumps(BREACHREALREQUEST, sort_keys=True)) BR_REAL_REP = {"feed.name": "HIBP Enterprise", "time.observation": "2019-03-01T01:01:01+00:00", diff --git a/intelmq/tests/lib/test_message.py b/intelmq/tests/lib/test_message.py index ac1e1cbcdd..f1269a9bb7 100644 --- a/intelmq/tests/lib/test_message.py +++ b/intelmq/tests/lib/test_message.py @@ -13,14 +13,12 @@ import json import unittest -import pkg_resources - import intelmq.lib.exceptions as exceptions import intelmq.lib.message as message # noqa -from intelmq.lib.utils import load_configuration +from intelmq.lib.utils import load_configuration, package_resource_path -HARM = load_configuration(pkg_resources.resource_filename('intelmq', - 'etc/harmonization.conf')) +HARM = load_configuration(package_resource_path('intelmq', + 'etc/harmonization.conf')) LOREM_BASE64 = 'bG9yZW0gaXBzdW0=' DOLOR_BASE64 = 'ZG9sb3Igc2l0IGFtZXQ=' diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index 72dbb1f592..19282810c2 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -6,11 +6,10 @@ Tests the upgrade functions. """ import unittest -import pkg_resources from copy import deepcopy import intelmq.lib.upgrades as upgrades -from intelmq.lib.utils import load_configuration +from intelmq.lib.utils import load_configuration, package_resource_path V202 = {"global": {}, @@ -261,8 +260,8 @@ "module": "intelmq.bots.collectors.misp.collector", "parameters": { "http_verify_cert": False}}} -HARM = load_configuration(pkg_resources.resource_filename('intelmq', - 'etc/harmonization.conf')) +HARM = load_configuration(package_resource_path('intelmq', + 'etc/harmonization.conf')) V210_HARM = deepcopy(HARM) del V210_HARM['report']['extra'] MISSING_REPORT = deepcopy(HARM) diff --git a/intelmq/tests/lib/test_utils.py b/intelmq/tests/lib/test_utils.py index ddb34408a3..c9b63b86b3 100644 --- a/intelmq/tests/lib/test_utils.py +++ b/intelmq/tests/lib/test_utils.py @@ -22,7 +22,6 @@ import cerberus import dns.resolver -import pkg_resources import requests import termstyle from ruamel.yaml.scanner import ScannerError @@ -51,7 +50,7 @@ def new_get_runtime() -> dict: - runtime_conf = utils.load_configuration(pkg_resources.resource_filename('intelmq', 'etc/runtime.yaml')) + runtime_conf = utils.load_configuration(utils.package_resource_path('intelmq', 'etc/runtime.yaml')) if 'global' not in runtime_conf: runtime_conf['global'] = {} runtime_conf['global']['http_proxy'] = 'http://localhost:8080' diff --git a/intelmq/tests/test_conf.py b/intelmq/tests/test_conf.py index 76e09a63e8..60bbdc06af 100644 --- a/intelmq/tests/test_conf.py +++ b/intelmq/tests/test_conf.py @@ -17,14 +17,13 @@ import unittest import cerberus -import pkg_resources from ruamel.yaml import YAML import intelmq.bots import intelmq.lib.harmonization as harmonization -from intelmq.lib.utils import lazy_int +from intelmq.lib.utils import lazy_int, package_resource_path yaml = YAML(typ="safe", pure=True) yaml.default_flow_style = False # This makes the configuration more readable, diff-friendly and consistent. @@ -46,8 +45,8 @@ def to_unsorted_json(obj): separators=(',', ': ')) + '\n' -CONF_FILES = {'harmonization': pkg_resources.resource_filename('intelmq', 'etc/harmonization.conf'), - 'runtime': pkg_resources.resource_filename('intelmq', 'etc/runtime.yaml')} +CONF_FILES = {'harmonization': package_resource_path('intelmq', 'etc/harmonization.conf'), + 'runtime': package_resource_path('intelmq', 'etc/runtime.yaml')} class TestConf(unittest.TestCase): @@ -113,8 +112,7 @@ def convert_cerberus_schema(self, schema: str) -> str: def test_feeds(self): with open(os.path.join(os.path.dirname(__file__), 'assets/feeds.schema.json')) as handle: schema = json.loads(self.convert_cerberus_schema(handle.read())) - with open(pkg_resources.resource_filename('intelmq', - 'etc/feeds.yaml'), encoding='UTF-8') as handle: + with open(package_resource_path('intelmq', 'etc/feeds.yaml'), encoding='UTF-8') as handle: feeds = yaml.load(handle) v = cerberus.Validator(schema)