diff --git a/bdpy/bdata/bdata.py b/bdpy/bdata/bdata.py index 0dc1ed2..07c3182 100644 --- a/bdpy/bdata/bdata.py +++ b/bdpy/bdata/bdata.py @@ -9,7 +9,6 @@ import datetime import functools -import inspect import os import re import time @@ -142,8 +141,8 @@ def metadata(self, value: MetaData) -> None: def header(self) -> Dict[str, Any]: """Header information associated with the BData instance. - The header stores auxiliary information such as creation time, - call stack, and values loaded from BData files. + The header stores auxiliary information such as creation time + and values loaded from BData files. Header keys are strings, while values are implementation-defined and may include strings, numbers, lists, or values loaded from external files. @@ -823,33 +822,17 @@ def load(self, load_filename: str, load_type: Optional[str] = None) -> None: def save(self, file_name: str, file_type: Optional[str] = None) -> None: """Save 'dataset' and 'metadata' to a file.""" - # Store data creation information + # Store data creation information. + # Note: older versions of bdpy also embedded the call stack (absolute + # file paths) and the full source code of every file on it into the + # header. That can leak sensitive information (internal paths, + # unpublished code) when BData files are shared, so it is no longer + # collected; only the creation time is recorded. t_now = time.time() t_now_str = datetime.datetime.fromtimestamp(t_now).strftime('%Y-%m-%d %H:%M:%S') - callstack = [] - callstack_code = [] - f = inspect.currentframe() - if f is None: - raise RuntimeError('Failed to get the current frame for call stack information.') - while True: - f = f.f_back - if f is None: - break - fname = os.path.abspath(f.f_code.co_filename) - fline = f.f_lineno - callstack.append('%s:%d' % (fname, fline)) - if os.path.exists(fname): - with open(fname, 'r') as fl: - fcode = fl.read() - else: - fcode = '' - callstack_code.append(fcode) - self.__header.update({'ctime': t_now_str, - 'ctime_epoch': t_now, - 'callstack': callstack, - 'callstack_code': callstack_code}) + 'ctime_epoch': t_now}) if file_type is None: file_type = self.__get_filetype(file_name) @@ -905,6 +888,8 @@ def __get_top_elm_from_order(self, order: np.ndarray, n: int) -> np.ndarray: def __save_h5(self, file_name: str, header: Optional[Dict[str, Any]] = None) -> None: """Save data in HDF5 format (*.h5).""" + legacy_header_keys = ('callstack', 'callstack_code') + with h5py.File(file_name, 'w') as h5file: # dataset h5file.create_dataset('/dataset', data=self.dataset) @@ -921,8 +906,26 @@ def __save_h5(self, file_name: str, header: Optional[Dict[str, Any]] = None) -> # header if header is not None: + # Omit legacy call-stack fields (absolute paths + full source + # code of the call stack) from the saved file for privacy. + # Neither self.__header nor the passed-in header dict is + # modified -- only the written copy is filtered. + legacy_keys = [k for k in legacy_header_keys if k in header] + if legacy_keys: + warnings.warn( + 'Omitting legacy header field(s) {} from the saved file for privacy.' + .format(', '.join(legacy_keys)), + UserWarning, + stacklevel=3, + ) + + header_to_write = { + k: v for k, v in header.items() + if k not in legacy_header_keys + } + h5file.create_group('/header') - for header_key, header_value in header.items(): + for header_key, header_value in header_to_write.items(): if isinstance(header_value, list): h5file.create_dataset( '/header/' + header_key, diff --git a/tests/bdata/test_bdata.py b/tests/bdata/test_bdata.py index fe20a65..78005a4 100644 --- a/tests/bdata/test_bdata.py +++ b/tests/bdata/test_bdata.py @@ -6,6 +6,7 @@ import unittest import warnings +import h5py import numpy as np from numpy.testing import assert_array_equal @@ -262,6 +263,71 @@ def test_save_load_hdf5_header_and_vmap(self): self.assertEqual(loaded_bdata.header['indices'], [1, 2]) self.assertEqual(loaded_bdata.header['scale'], 1.5) + def test_save_no_callstack_header(self): + '''save() records creation time but no call-stack information.''' + bdata = BData() + bdata.add(np.arange(6, dtype=float).reshape(3, 2), 'Data') + + with tempfile.TemporaryDirectory() as temp_dir: + h5_path = os.path.join(temp_dir, 'test_bdata.h5') + bdata.save(h5_path, 'HDF5') + loaded_bdata = BData(h5_path, 'HDF5') + + self.assertIn('ctime', loaded_bdata.header) + self.assertIn('ctime_epoch', loaded_bdata.header) + self.assertNotIn('callstack', loaded_bdata.header) + self.assertNotIn('callstack_code', loaded_bdata.header) + + def test_save_excludes_legacy_callstack_from_file(self): + '''save() omits legacy call-stack fields from the file but keeps them in memory.''' + bdata = BData() + bdata.add(np.arange(6, dtype=float).reshape(3, 2), 'Data') + bdata.update_header({ + 'source': 'manual', + 'callstack': ['/abs/path/to/script.py:42'], + 'callstack_code': ['secret source code'], + }) + + with tempfile.TemporaryDirectory() as temp_dir: + h5_path = os.path.join(temp_dir, 'test_bdata.h5') + with self.assertWarns(UserWarning): + bdata.save(h5_path, 'HDF5') + loaded_bdata = BData(h5_path, 'HDF5') + + # Legacy fields are omitted from the saved file, but the in-memory + # header is left untouched; intentionally set fields survive both. + self.assertNotIn('callstack', loaded_bdata.header) + self.assertNotIn('callstack_code', loaded_bdata.header) + self.assertEqual(loaded_bdata.header['source'], 'manual') + self.assertIn('callstack', bdata.header) + self.assertIn('callstack_code', bdata.header) + self.assertEqual(bdata.header['source'], 'manual') + + def test_save_h5_header_none_keeps_memory(self): + '''__save_h5(header=None) writes no header group and leaves memory intact.''' + bdata = BData() + bdata.add(np.arange(6, dtype=float).reshape(3, 2), 'Data') + bdata.update_header({ + 'source': 'manual', + 'callstack': ['/abs/path/to/script.py:42'], + 'callstack_code': ['secret source code'], + }) + + with tempfile.TemporaryDirectory() as temp_dir: + h5_path = os.path.join(temp_dir, 'test_bdata.h5') + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + bdata._BData__save_h5(h5_path, header=None) + with h5py.File(h5_path, 'r') as f: + self.assertNotIn('header', f) + + # No header is written, so nothing is omitted and no warning fires. + self.assertFalse([w for w in caught if issubclass(w.category, UserWarning)]) + # The in-memory header is left fully intact. + self.assertIn('callstack', bdata.header) + self.assertIn('callstack_code', bdata.header) + self.assertEqual(bdata.header['source'], 'manual') + # Tests for vmap def test_vmap_add_get(self): bdata = BData()