From 76bc4be9c923779b67fda076f82df67a5d22632c Mon Sep 17 00:00:00 2001 From: Kenya Otsuka Date: Fri, 26 Jun 2026 01:01:40 +0000 Subject: [PATCH 1/3] fix(bdata): stop embedding call stack and source code in saved headers BData.save() previously recorded the full call stack (absolute file paths) and the entire source code of every file on it into the saved file's header. This is undocumented and can leak internal paths and unpublished code when BData files are shared or published. Stop collecting `callstack` / `callstack_code`, keeping only `ctime` / `ctime_epoch`. To avoid re-writing legacy fields restored on load, strip any `callstack` / `callstack_code` from the header on save and emit a UserWarning so the change is visible. Closes #131. --- bdpy/bdata/bdata.py | 48 ++++++++++++++++++--------------------- tests/bdata/test_bdata.py | 39 +++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 26 deletions(-) diff --git a/bdpy/bdata/bdata.py b/bdpy/bdata/bdata.py index 0dc1ed2..8e07522 100644 --- a/bdpy/bdata/bdata.py +++ b/bdpy/bdata/bdata.py @@ -9,7 +9,6 @@ import datetime import functools -import inspect import os import re import time @@ -142,8 +141,8 @@ def metadata(self, value: MetaData) -> None: def header(self) -> Dict[str, Any]: """Header information associated with the BData instance. - The header stores auxiliary information such as creation time, - call stack, and values loaded from BData files. + The header stores auxiliary information such as creation time + and values loaded from BData files. Header keys are strings, while values are implementation-defined and may include strings, numbers, lists, or values loaded from external files. @@ -823,33 +822,30 @@ def load(self, load_filename: str, load_type: Optional[str] = None) -> None: def save(self, file_name: str, file_type: Optional[str] = None) -> None: """Save 'dataset' and 'metadata' to a file.""" - # Store data creation information + # Store data creation information. + # Note: older versions of bdpy also embedded the call stack (absolute + # file paths) and the full source code of every file on it into the + # header. That can leak sensitive information (internal paths, + # unpublished code) when BData files are shared, so it is no longer + # collected; only the creation time is recorded. t_now = time.time() t_now_str = datetime.datetime.fromtimestamp(t_now).strftime('%Y-%m-%d %H:%M:%S') - callstack = [] - callstack_code = [] - f = inspect.currentframe() - if f is None: - raise RuntimeError('Failed to get the current frame for call stack information.') - while True: - f = f.f_back - if f is None: - break - fname = os.path.abspath(f.f_code.co_filename) - fline = f.f_lineno - callstack.append('%s:%d' % (fname, fline)) - if os.path.exists(fname): - with open(fname, 'r') as fl: - fcode = fl.read() - else: - fcode = '' - callstack_code.append(fcode) - self.__header.update({'ctime': t_now_str, - 'ctime_epoch': t_now, - 'callstack': callstack, - 'callstack_code': callstack_code}) + 'ctime_epoch': t_now}) + + # Strip legacy call-stack fields that may have been loaded from a file + # saved by an older version, so they are not written out again. + legacy_keys = [k for k in ('callstack', 'callstack_code') if k in self.__header] + if legacy_keys: + warnings.warn( + "Removing legacy header field(s) %s on save for privacy; " + "they will not be written to the file." % ', '.join(legacy_keys), + UserWarning, + stacklevel=2, + ) + for k in legacy_keys: + del self.__header[k] if file_type is None: file_type = self.__get_filetype(file_name) diff --git a/tests/bdata/test_bdata.py b/tests/bdata/test_bdata.py index fe20a65..df0e9c1 100644 --- a/tests/bdata/test_bdata.py +++ b/tests/bdata/test_bdata.py @@ -262,6 +262,45 @@ def test_save_load_hdf5_header_and_vmap(self): self.assertEqual(loaded_bdata.header['indices'], [1, 2]) self.assertEqual(loaded_bdata.header['scale'], 1.5) + def test_save_no_callstack_header(self): + '''save() records creation time but no call-stack information.''' + bdata = BData() + bdata.add(np.arange(6, dtype=float).reshape(3, 2), 'Data') + + with tempfile.TemporaryDirectory() as temp_dir: + h5_path = os.path.join(temp_dir, 'test_bdata.h5') + bdata.save(h5_path, 'HDF5') + loaded_bdata = BData(h5_path, 'HDF5') + + self.assertIn('ctime', loaded_bdata.header) + self.assertIn('ctime_epoch', loaded_bdata.header) + self.assertNotIn('callstack', loaded_bdata.header) + self.assertNotIn('callstack_code', loaded_bdata.header) + + def test_save_strips_legacy_callstack(self): + '''save() drops legacy call-stack header fields and warns.''' + bdata = BData() + bdata.add(np.arange(6, dtype=float).reshape(3, 2), 'Data') + bdata.update_header({ + 'source': 'manual', + 'callstack': ['/abs/path/to/script.py:42'], + 'callstack_code': ['secret source code'], + }) + + with tempfile.TemporaryDirectory() as temp_dir: + h5_path = os.path.join(temp_dir, 'test_bdata.h5') + with self.assertWarns(UserWarning): + bdata.save(h5_path, 'HDF5') + loaded_bdata = BData(h5_path, 'HDF5') + + # Legacy fields are stripped both in memory and in the saved file, + # while intentionally set header fields survive. + self.assertNotIn('callstack', bdata.header) + self.assertNotIn('callstack_code', bdata.header) + self.assertNotIn('callstack', loaded_bdata.header) + self.assertNotIn('callstack_code', loaded_bdata.header) + self.assertEqual(loaded_bdata.header['source'], 'manual') + # Tests for vmap def test_vmap_add_get(self): bdata = BData() From d414b1d1b9f4fc143ed105f805cfd02f2909418c Mon Sep 17 00:00:00 2001 From: Kenya Otsuka Date: Fri, 26 Jun 2026 09:17:01 +0000 Subject: [PATCH 2/3] refactor(bdata): move legacy header stripping into __save_h5 Move the call-stack legacy-field stripping out of save() and into __save_h5() so it runs on every save path. Previously the documented private workaround `_BData__save_h5(path, header=None)` wrote no header group but left `callstack` / `callstack_code` in the in-memory header. Now that path also strips them (with the same UserWarning), keeping the in-memory header consistent regardless of how the data is saved. --- bdpy/bdata/bdata.py | 29 ++++++++++++++++------------- tests/bdata/test_bdata.py | 24 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/bdpy/bdata/bdata.py b/bdpy/bdata/bdata.py index 8e07522..0264a26 100644 --- a/bdpy/bdata/bdata.py +++ b/bdpy/bdata/bdata.py @@ -834,19 +834,6 @@ def save(self, file_name: str, file_type: Optional[str] = None) -> None: self.__header.update({'ctime': t_now_str, 'ctime_epoch': t_now}) - # Strip legacy call-stack fields that may have been loaded from a file - # saved by an older version, so they are not written out again. - legacy_keys = [k for k in ('callstack', 'callstack_code') if k in self.__header] - if legacy_keys: - warnings.warn( - "Removing legacy header field(s) %s on save for privacy; " - "they will not be written to the file." % ', '.join(legacy_keys), - UserWarning, - stacklevel=2, - ) - for k in legacy_keys: - del self.__header[k] - if file_type is None: file_type = self.__get_filetype(file_name) @@ -901,6 +888,22 @@ def __get_top_elm_from_order(self, order: np.ndarray, n: int) -> np.ndarray: def __save_h5(self, file_name: str, header: Optional[Dict[str, Any]] = None) -> None: """Save data in HDF5 format (*.h5).""" + # Remove legacy call-stack fields from the in-memory header so they are + # neither written out nor retained after a save. This runs on every + # save path, including the private `header=None` path used to skip the + # header group, so a file opened with an older version is sanitized + # regardless of how it is saved. + legacy_keys = [k for k in ('callstack', 'callstack_code') if k in self.__header] + if legacy_keys: + warnings.warn( + "Removing legacy header field(s) %s on save for privacy; " + "they will not be written to the file." % ', '.join(legacy_keys), + UserWarning, + stacklevel=3, + ) + for k in legacy_keys: + del self.__header[k] + with h5py.File(file_name, 'w') as h5file: # dataset h5file.create_dataset('/dataset', data=self.dataset) diff --git a/tests/bdata/test_bdata.py b/tests/bdata/test_bdata.py index df0e9c1..53dfbfd 100644 --- a/tests/bdata/test_bdata.py +++ b/tests/bdata/test_bdata.py @@ -6,6 +6,7 @@ import unittest import warnings +import h5py import numpy as np from numpy.testing import assert_array_equal @@ -301,6 +302,29 @@ def test_save_strips_legacy_callstack(self): self.assertNotIn('callstack_code', loaded_bdata.header) self.assertEqual(loaded_bdata.header['source'], 'manual') + def test_save_h5_header_none_strips_legacy_inplace(self): + '''__save_h5(header=None) writes no header group but still cleans memory.''' + bdata = BData() + bdata.add(np.arange(6, dtype=float).reshape(3, 2), 'Data') + bdata.update_header({ + 'source': 'manual', + 'callstack': ['/abs/path/to/script.py:42'], + 'callstack_code': ['secret source code'], + }) + + with tempfile.TemporaryDirectory() as temp_dir: + h5_path = os.path.join(temp_dir, 'test_bdata.h5') + with self.assertWarns(UserWarning): + bdata._BData__save_h5(h5_path, header=None) + with h5py.File(h5_path, 'r') as f: + self.assertNotIn('header', f) + + # Legacy fields are stripped from the in-memory header even though no + # header group was written, while other fields are kept. + self.assertNotIn('callstack', bdata.header) + self.assertNotIn('callstack_code', bdata.header) + self.assertEqual(bdata.header['source'], 'manual') + # Tests for vmap def test_vmap_add_get(self): bdata = BData() From 45df2d17276afde0b277bd26cdfc73dc66a8ce91 Mon Sep 17 00:00:00 2001 From: Kenya Otsuka Date: Fri, 26 Jun 2026 11:23:31 +0000 Subject: [PATCH 3/3] fix(bdata): exclude legacy header fields on write without mutating memory Previously the legacy call-stack stripping deleted `callstack` / `callstack_code` from the in-memory header (`self.__header`). Issue #131 only asks to stop *writing* those fields to files, and silently destroying in-memory data the user holds is an over-reach. Filter the legacy fields out of a copy used for writing instead, leaving both `self.__header` and the passed-in `header` dict untouched. The `header=None` path writes no header group and emits no warning. A UserWarning is raised only when legacy fields are actually omitted from a written header. --- bdpy/bdata/bdata.py | 36 ++++++++++++++++++++---------------- tests/bdata/test_bdata.py | 29 ++++++++++++++++------------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/bdpy/bdata/bdata.py b/bdpy/bdata/bdata.py index 0264a26..07c3182 100644 --- a/bdpy/bdata/bdata.py +++ b/bdpy/bdata/bdata.py @@ -888,21 +888,7 @@ def __get_top_elm_from_order(self, order: np.ndarray, n: int) -> np.ndarray: def __save_h5(self, file_name: str, header: Optional[Dict[str, Any]] = None) -> None: """Save data in HDF5 format (*.h5).""" - # Remove legacy call-stack fields from the in-memory header so they are - # neither written out nor retained after a save. This runs on every - # save path, including the private `header=None` path used to skip the - # header group, so a file opened with an older version is sanitized - # regardless of how it is saved. - legacy_keys = [k for k in ('callstack', 'callstack_code') if k in self.__header] - if legacy_keys: - warnings.warn( - "Removing legacy header field(s) %s on save for privacy; " - "they will not be written to the file." % ', '.join(legacy_keys), - UserWarning, - stacklevel=3, - ) - for k in legacy_keys: - del self.__header[k] + legacy_header_keys = ('callstack', 'callstack_code') with h5py.File(file_name, 'w') as h5file: # dataset @@ -920,8 +906,26 @@ def __save_h5(self, file_name: str, header: Optional[Dict[str, Any]] = None) -> # header if header is not None: + # Omit legacy call-stack fields (absolute paths + full source + # code of the call stack) from the saved file for privacy. + # Neither self.__header nor the passed-in header dict is + # modified -- only the written copy is filtered. + legacy_keys = [k for k in legacy_header_keys if k in header] + if legacy_keys: + warnings.warn( + 'Omitting legacy header field(s) {} from the saved file for privacy.' + .format(', '.join(legacy_keys)), + UserWarning, + stacklevel=3, + ) + + header_to_write = { + k: v for k, v in header.items() + if k not in legacy_header_keys + } + h5file.create_group('/header') - for header_key, header_value in header.items(): + for header_key, header_value in header_to_write.items(): if isinstance(header_value, list): h5file.create_dataset( '/header/' + header_key, diff --git a/tests/bdata/test_bdata.py b/tests/bdata/test_bdata.py index 53dfbfd..78005a4 100644 --- a/tests/bdata/test_bdata.py +++ b/tests/bdata/test_bdata.py @@ -278,8 +278,8 @@ def test_save_no_callstack_header(self): self.assertNotIn('callstack', loaded_bdata.header) self.assertNotIn('callstack_code', loaded_bdata.header) - def test_save_strips_legacy_callstack(self): - '''save() drops legacy call-stack header fields and warns.''' + def test_save_excludes_legacy_callstack_from_file(self): + '''save() omits legacy call-stack fields from the file but keeps them in memory.''' bdata = BData() bdata.add(np.arange(6, dtype=float).reshape(3, 2), 'Data') bdata.update_header({ @@ -294,16 +294,17 @@ def test_save_strips_legacy_callstack(self): bdata.save(h5_path, 'HDF5') loaded_bdata = BData(h5_path, 'HDF5') - # Legacy fields are stripped both in memory and in the saved file, - # while intentionally set header fields survive. - self.assertNotIn('callstack', bdata.header) - self.assertNotIn('callstack_code', bdata.header) + # Legacy fields are omitted from the saved file, but the in-memory + # header is left untouched; intentionally set fields survive both. self.assertNotIn('callstack', loaded_bdata.header) self.assertNotIn('callstack_code', loaded_bdata.header) self.assertEqual(loaded_bdata.header['source'], 'manual') + self.assertIn('callstack', bdata.header) + self.assertIn('callstack_code', bdata.header) + self.assertEqual(bdata.header['source'], 'manual') - def test_save_h5_header_none_strips_legacy_inplace(self): - '''__save_h5(header=None) writes no header group but still cleans memory.''' + def test_save_h5_header_none_keeps_memory(self): + '''__save_h5(header=None) writes no header group and leaves memory intact.''' bdata = BData() bdata.add(np.arange(6, dtype=float).reshape(3, 2), 'Data') bdata.update_header({ @@ -314,15 +315,17 @@ def test_save_h5_header_none_strips_legacy_inplace(self): with tempfile.TemporaryDirectory() as temp_dir: h5_path = os.path.join(temp_dir, 'test_bdata.h5') - with self.assertWarns(UserWarning): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') bdata._BData__save_h5(h5_path, header=None) with h5py.File(h5_path, 'r') as f: self.assertNotIn('header', f) - # Legacy fields are stripped from the in-memory header even though no - # header group was written, while other fields are kept. - self.assertNotIn('callstack', bdata.header) - self.assertNotIn('callstack_code', bdata.header) + # No header is written, so nothing is omitted and no warning fires. + self.assertFalse([w for w in caught if issubclass(w.category, UserWarning)]) + # The in-memory header is left fully intact. + self.assertIn('callstack', bdata.header) + self.assertIn('callstack_code', bdata.header) self.assertEqual(bdata.header['source'], 'manual') # Tests for vmap