From 13e5fa0acf8506a213c9a370f41cab0b08869edf Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 28 Jul 2025 12:40:49 +0200 Subject: [PATCH 1/4] Change config --- faassupervisor/storage/providers/rucio.py | 52 +++++++++++++---------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/faassupervisor/storage/providers/rucio.py b/faassupervisor/storage/providers/rucio.py index 3402ff5..50da5c2 100644 --- a/faassupervisor/storage/providers/rucio.py +++ b/faassupervisor/storage/providers/rucio.py @@ -25,6 +25,7 @@ pass +from rucio.common.config import config_set, config_add_section from rucio.client.client import Client from rucio.client.uploadclient import UploadClient from rucio.client.downloadclient import DownloadClient @@ -49,10 +50,6 @@ def __init__(self, stg_auth): if not self.client_id: self.client_id = 'token-portal' self.oidc_token = self.stg_auth.get_credential('access_token') - self.token_temp_file = tempfile.NamedTemporaryFile(delete=False) - self.token_temp_file.close() - self.cfg_temp_file = tempfile.NamedTemporaryFile(delete=False) - self.cfg_temp_file.close() self.rucio_host = self.stg_auth.get_credential('host') self.auth_host = self.stg_auth.get_credential('auth_host') self.scope = self.stg_auth.get_credential('account') @@ -64,12 +61,15 @@ def __init__(self, stg_auth): self.token_endpoint = OIDCUtils.DEFAULT_TOKEN_ENDPOINT self.scopes = self.stg_auth.get_credential('scopes') if not self.scopes: - self.scopes = OIDCUtils.DEFAULT_SCOPES + self.scopes = self._OIDC_SCOPE.split() + self.token_temp_file = tempfile.mktemp(prefix='rucio_token_', + suffix='.token') # nosec + self._create_rucio_config() def __del__(self): try: - os.remove(self.token_temp_file.name) - os.remove(self.cfg_temp_file.name) + if os.path.exists(self.token_temp_file): + os.remove(self.token_temp_file) except Exception as exc: get_logger().warning('Error removing temporary file: %s', exc) @@ -87,20 +87,21 @@ def _get_access_token(self): self.client_id) return self.oidc_token + def _create_rucio_config(self): + os.environ["RUCIO_CLIENT_MODE"] = "1" + os.environ["RUCIO_CONFIG"] = "/dev/null" + config_add_section("client") + config_set(section="client", option="auth_token_file_path", value=self.token_temp_file) + config_set(section="client", option="oidc_scope", value=self._OIDC_SCOPE) + config_set(section="client", option="rucio_host", value=self.rucio_host) + config_set(section="client", option="auth_host", value=self.auth_host) + config_set(section="client", option="account", value=self.scope) + config_set(section="client", option="auth_type", value="oidc") + def _get_rucio_client(self, client_type=None): # Create token file - with open(self.token_temp_file.name, 'w') as f: - f.write(self.oidc_token) - # Create config file - with open(self.cfg_temp_file.name, 'w') as f: - f.write('[client]\n') - f.write('rucio_host = %s\n' % self.rucio_host) - f.write('auth_host = %s\n' % self.auth_host) - f.write('auth_type = oidc\n') - f.write('account = %s\n' % self.scope) - f.write('auth_token_file_path = %s\n' % self.token_temp_file.name) - f.write('oidc_scope = %s\n' % self._OIDC_SCOPE) - os.environ['RUCIO_CONFIG'] = self.cfg_temp_file.name + with open(self.token_temp_file, 'w') as f: + f.write(self._get_access_token()) client = Client() if not client_type: return client @@ -123,13 +124,18 @@ def download_file(self, parsed_event, input_dir_path): dids = [{'did': '%s:%s' % (f['scope'], f['name'])} for f in files] - output_dir = SysUtils.join_paths(input_dir_path, dataset_name) - FileUtils.create_folder(output_dir) - downloadc = self._get_rucio_client("download") download = downloadc.download_dids(dids) get_logger().debug('Downloaded file info: %s', download) - return output_dir + try: + for file in download: + complete_name = file["dest_file_paths"][0] + basename = os.path.basename(complete_name) + FileUtils.cp_file(complete_name, SysUtils.join_paths(input_dir_path, + basename)) + except Exception as e: + print("An exception occurred" + e) + return input_dir_path def upload_file(self, file_path, file_name, output_path): """Uploads the file to Rucio. From e6adb2b463a3b3fb75ae1da4f672e092a69d75d0 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 28 Jul 2025 12:45:29 +0200 Subject: [PATCH 2/4] Fix error --- faassupervisor/storage/providers/rucio.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/faassupervisor/storage/providers/rucio.py b/faassupervisor/storage/providers/rucio.py index 50da5c2..c309153 100644 --- a/faassupervisor/storage/providers/rucio.py +++ b/faassupervisor/storage/providers/rucio.py @@ -25,7 +25,7 @@ pass -from rucio.common.config import config_set, config_add_section +from rucio.common.config import config_set, config_add_section, config_has_section from rucio.client.client import Client from rucio.client.uploadclient import UploadClient from rucio.client.downloadclient import DownloadClient @@ -90,7 +90,9 @@ def _get_access_token(self): def _create_rucio_config(self): os.environ["RUCIO_CLIENT_MODE"] = "1" os.environ["RUCIO_CONFIG"] = "/dev/null" - config_add_section("client") + + if not config_has_section("client"): + config_add_section("client") config_set(section="client", option="auth_token_file_path", value=self.token_temp_file) config_set(section="client", option="oidc_scope", value=self._OIDC_SCOPE) config_set(section="client", option="rucio_host", value=self.rucio_host) From 31f5c8a6f2bd4413cfba79c6098e447617147e18 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 28 Jul 2025 12:47:17 +0200 Subject: [PATCH 3/4] Fix error --- test/unit/faassupervisor/storage.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/unit/faassupervisor/storage.py b/test/unit/faassupervisor/storage.py index f117d94..ed6360c 100644 --- a/test/unit/faassupervisor/storage.py +++ b/test/unit/faassupervisor/storage.py @@ -566,10 +566,6 @@ def test_download_file(self, mock_cp, mock_refesh, mock_download, mock_client): mock_download_client.download_dids.assert_called_once_with([{'did': 'test_account2:file1'}, {'did': 'test_account2:file2'}]) mock_rucio_client.list_files.assert_called_once_with('test_account2', 'dataset_name') - with open(rucio_provider.cfg_temp_file, 'r') as f: - content = f.read() - exp_content = RUCIO_CFG_FILE.format(rucio_provider=rucio_provider) - self.assertEqual(content, exp_content) with open(rucio_provider.token_temp_file, 'r') as f: content = f.read() self.assertEqual(content, 'new_access_token') From d3e10936dc281e39b6d297e698ff255d47e40999 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 28 Jul 2025 12:59:23 +0200 Subject: [PATCH 4/4] Improve test --- test/unit/faassupervisor/storage.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/unit/faassupervisor/storage.py b/test/unit/faassupervisor/storage.py index ed6360c..83d169a 100644 --- a/test/unit/faassupervisor/storage.py +++ b/test/unit/faassupervisor/storage.py @@ -31,6 +31,7 @@ from faassupervisor.events.onedata import OnedataEvent from faassupervisor.utils import StrUtils from rucio.common.exception import DataIdentifierNotFound +from rucio.common.config import config_get, config_has_section # pylint: disable=missing-docstring @@ -569,6 +570,13 @@ def test_download_file(self, mock_cp, mock_refesh, mock_download, mock_client): with open(rucio_provider.token_temp_file, 'r') as f: content = f.read() self.assertEqual(content, 'new_access_token') + self.assertTrue(config_has_section('client')) + self.assertEqual(config_get('client', 'auth_token_file_path'), rucio_provider.token_temp_file) + self.assertEqual(config_get('client', 'oidc_scope'), Rucio._OIDC_SCOPE) + self.assertEqual(config_get('client', 'rucio_host'), rucio_provider.rucio_host) + self.assertEqual(config_get('client', 'auth_host'), rucio_provider.auth_host) + self.assertEqual(config_get('client', 'account'), rucio_provider.scope) + self.assertEqual(config_get('client', 'auth_type'), 'oidc') @mock.patch('faassupervisor.storage.providers.rucio.UploadClient') @mock.patch('faassupervisor.storage.providers.rucio.Client')