formatting fixes (#15)

Author: jacobfilik

swmr_tools/datasource.py

@@ -6,6 +6,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 class DataSource:
     """Iterator for returning dataset frames for any number of datasets. This
     class is largely a wrapper around the KeyFollower and FrameReader classes.
@@ -87,45 +88,42 @@ def create_dataset(self, data, fh, path):
 
         scan_max = self.kf.maxshape
         maxshape = scan_max + data.shape
-        shape = ([1] * len(scan_max) + list(data.shape))
+        shape = [1] * len(scan_max) + list(data.shape)
         r = data.reshape(shape)
-        return fh.create_dataset(path, data=r, maxshape = maxshape)
+        return fh.create_dataset(path, data=r, maxshape=maxshape)
 
     def append_data(self, data, slice_metadata, dataset):
-        ds = tuple(slice(0,s,1) for s in data.shape)
+        ds = tuple(slice(0, s, 1) for s in data.shape)
         fullslice = slice_metadata + ds
         current = dataset.shape
-        new_shape = tuple(max(s.stop,c) for (s,c) in zip(fullslice,current))
-        if (np.any(new_shape > current)):
+        new_shape = tuple(max(s.stop, c) for (s, c) in zip(fullslice, current))
+        if np.any(new_shape > current):
             dataset.resize(new_shape)
         dataset[fullslice] = data
 
     @staticmethod
-    def check_file_readable(path, datasets,timeout = 10, retrys = 5):
+    def check_file_readable(path, datasets, timeout=10, retrys=5):
         start = time.time()
         dif = time.time() - start
 
         while dif < timeout:
             try:
-                with h5py.File(path,'r',libver = "latest", swmr = True) as fh:
+                with h5py.File(path, "r", libver="latest", swmr=True) as fh:
                     for d in datasets:
-                        tmp = fh[d]
+                        fh[d]
                     return True
 
             except Exception as e:
                 logger.debug("Reading failed, retrying " + str(e))
-                time.sleep(timeout/retrys)
+                time.sleep(timeout / retrys)
                 dif = time.time() - start
 
         logger.error("Could not read file " + path)
         return False
 
 
-
-
 class SliceDict(dict):
-    """Dictionary with attributes for the slice metadata and maxshape of the scan
-    """
+    """Dictionary with attributes for the slice metadata and maxshape of the scan"""
 
     def __init__(self, *args, **kw):
         super(SliceDict, self).__init__(*args, **kw)
@@ -203,10 +201,10 @@ def read_frame(self, index):
         shape = ds.shape
 
         try:
-            #might fail if dataset is cached
+            # might fail if dataset is cached
             pos, slices, shape_slice = self.get_pos(index, shape)
         except ValueError:
-            #refresh dataset and try again
+            # refresh dataset and try again
             if hasattr(ds, "refresh"):
                 ds.refresh()
 
@@ -232,4 +230,3 @@ def get_pos(self, index, shape):
         pos = np.unravel_index(index, scan_shape)
 
         return pos, slices, shape_slice
-

swmr_tools/keyfollower.py

@@ -26,7 +26,7 @@ class KeyFollower:
 
     finished_dataset: string (optional)
         Path to a scalar hdf5 dataset which is zero when the file is being
-        written to and non-zero when the file is complete. Used to stop 
+        written to and non-zero when the file is complete. Used to stop
         the iterator without waiting for the timeout
 
 
@@ -86,7 +86,7 @@ def check_datasets(self):
             if rank != -1 and rank != r:
                 raise RuntimeError("Key datasets must have the same rank!")
 
-            if (self.maxshape is None):
+            if self.maxshape is None:
                 self.maxshape = tmp.maxshape[:rank]
             else:
                 if np.all(self.maxshape != tmp.maxshape[:rank]):
@@ -171,7 +171,7 @@ def _is_next(self):
         new_max = np.argmax(merged == 0) - 1
 
         if new_max < 0 and merged[0] != 0:
-            #all keys non zero
+            # all keys non zero
             new_max = merged.size - 1
 
         if self.current_max == new_max:
@@ -211,9 +211,9 @@ def _check_finished_dataset(self):
         if self._prelim_finished_check and finished:
             return True
 
-        #go through the timeout loop once more
-        #in case finish is flushed slightly before
-        #the last of the keys
+        # go through the timeout loop once more
+        # in case finish is flushed slightly before
+        # the last of the keys
         if finished:
             self._prelim_finished_check = True
 

tests/test_DataSource.py

@@ -24,8 +24,7 @@ def test_iterates_complete_dataset():
         d = dset["data/complete"]
         assert dset.maxshape == [10]
         assert dset.index == val
-        assert dset.slice_metadata == (slice(val,val+1,None),)
+        assert dset.slice_metadata == (slice(val, val + 1, None),)
         print(dset.slice_metadata)
         assert d == val
         val = val + 1
-

tests/test_KeyFollower.py

@@ -4,16 +4,14 @@
 
 def test_first_frame():
 
-    key_paths =  ["k1", "k2", "k3"]
-    shape=[10]
-
+    key_paths = ["k1", "k2", "k3"]
+    shape = [10]
 
     k1 = utils.make_mock(shape)
     k2 = utils.make_mock(shape)
     k3 = utils.make_mock(shape)
 
-
-    f = {"k1": k1, "k2" : k2, "k3": k3}
+    f = {"k1": k1, "k2": k2, "k3": k3}
     kf = KeyFollower(f, key_paths, timeout=0.1)
     kf.check_datasets()
 
@@ -46,7 +44,6 @@ def test_first_frame():
 
     assert current_key == 0
 
-
     kf.reset()
 
     k1.dataset[1] = 1
@@ -70,14 +67,13 @@ def test_first_frame():
 
 def test_first_frame_jagged():
 
-    key_paths =  ["k1", "k2", "k3"]
+    key_paths = ["k1", "k2", "k3"]
 
     k1 = utils.make_mock([2])
     k2 = utils.make_mock([2])
-    k3 = utils.make_mock([1],maxshape=[2])
+    k3 = utils.make_mock([1], maxshape=[2])
 
-
-    f = {"k1": k1, "k2" : k2, "k3": k3}
+    f = {"k1": k1, "k2": k2, "k3": k3}
     kf = KeyFollower(f, key_paths, timeout=0.1)
     kf.check_datasets()
 
@@ -111,6 +107,7 @@ def test_first_frame_jagged():
 
     assert current_key == 0
 
+
 def test_iterates_complete_dataset():
 
     key_paths = ["complete"]

tests/test_using_h5py.py

@@ -1,7 +1,6 @@
 import h5py
 import numpy as np
 import time
-import os
 import multiprocessing as mp
 from swmr_tools import KeyFollower, DataSource
 from functools import reduce
@@ -31,6 +30,7 @@ def test_multiple_keys(tmp_path):
 
         assert current_key == 5
 
+
 def test_complete_keys(tmp_path):
 
     f = str(tmp_path / "f.h5")
@@ -54,6 +54,7 @@ def test_complete_keys(tmp_path):
 
         assert current_key == 10
 
+
 def test_data_read(tmp_path):
 
     f = str(tmp_path / "f.h5")
@@ -74,6 +75,7 @@ def test_data_read(tmp_path):
             assert np.all(d == base + (20 * count))
             count = count + 1
 
+
 def test_use_case_example(tmp_path):
 
     f = str(tmp_path / "f.h5")
@@ -94,32 +96,33 @@ def test_use_case_example(tmp_path):
         for dset in df:
             d = dset["/data"]
             d = d.squeeze()
-            r = d.sum(axis = 1)
-            assert dset.maxshape == (2,3)
+            r = d.sum(axis=1)
+            assert dset.maxshape == (2, 3)
 
             if output is None:
-                output = df.create_dataset(r,oh,output_path)
+                output = df.create_dataset(r, oh, output_path)
             else:
-                df.append_data(r,dset.slice_metadata,output)
+                df.append_data(r, dset.slice_metadata, output)
 
     with h5py.File(o, "r") as oh:
         out = oh["/result"]
-        assert out.shape == (2,3,4)
-        assert out.maxshape == (2,3,4)
+        assert out.shape == (2, 3, 4)
+        assert out.maxshape == (2, 3, 4)
         print(out[...])
-        assert 119+118+117+116+115 == out[1,2,3]
-        assert out[0,1,0] != 0
+        assert 119 + 118 + 117 + 116 + 115 == out[1, 2, 3]
+        assert out[0, 1, 0] != 0
+
 
 def test_mock_scan(tmp_path):
     f = str(tmp_path / "scan.h5")
 
-    mp.set_start_method('spawn')
-    p = mp.Process(target=mock_scan, args = (f,))
+    mp.set_start_method("spawn")
+    p = mp.Process(target=mock_scan, args=(f,))
     p.start()
 
-    DataSource.check_file_readable(f, ["/data","/key"], timeout = 5)
+    DataSource.check_file_readable(f, ["/data", "/key"], timeout=5)
 
-    with h5py.File(f, "r", libver = "latest", swmr = True) as fh:
+    with h5py.File(f, "r", libver="latest", swmr=True) as fh:
 
         data_paths = ["/data"]
         key_paths = ["/key"]
@@ -130,11 +133,12 @@ def test_mock_scan(tmp_path):
         assert p.is_alive()
         for dset in df:
             d = dset["/data"]
-            assert d[0,0,0].item() == count
+            assert d[0, 0, 0].item() == count
             count = count + 1
 
     p.join()
 
+
 def create_test_file(path):
 
     with h5py.File(path, "w") as fh:
@@ -149,7 +153,7 @@ def create_test_file(path):
 
 def mock_scan(path):
 
-    with h5py.File(path, "w", libver = "latest") as fh:
+    with h5py.File(path, "w", libver="latest") as fh:
         maxn = 10
         maxshape = (maxn, 9, 10)
         shape = (1, 9, 10)
@@ -161,14 +165,13 @@ def mock_scan(path):
         ks = fh.create_dataset("key", data=k, maxshape=(20,))
         fh.swmr_mode = True
         for i in range(maxn):
-            s = (i+1, 9, 10)
+            s = (i + 1, 9, 10)
             if i != 0:
                 ds.resize(s)
-                ks.resize((i+1,))
-            ds[i,:,:] = np.ones(shape) + i
+                ks.resize((i + 1,))
+            ds[i, :, :] = np.ones(shape) + i
             ds.flush()
             ks[i] = 1
             ks.flush()
             print("flush " + str(i))
             time.sleep(1)
-

tests/utils.py

@@ -2,7 +2,7 @@
 import numpy as np
 
 
-def make_mock(shape=[5, 10, 1, 1],maxshape=None):
+def make_mock(shape=[5, 10, 1, 1], maxshape=None):
 
     mds = Mock()
     mds.dataset = np.zeros(shape)