From 6150971b5e6333c1a8bbcfde403cb94b624868e7 Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Tue, 31 Dec 2019 18:58:56 -0800 Subject: [PATCH 1/6] SoundFileLoader: Add experimental (untested) spectrogram support --- ultratrace2/model/files/loaders/base.py | 16 +++++++ ultratrace2/model/files/loaders/pydub.py | 54 +++++++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/ultratrace2/model/files/loaders/base.py b/ultratrace2/model/files/loaders/base.py index 99b4b6e..8305748 100644 --- a/ultratrace2/model/files/loaders/base.py +++ b/ultratrace2/model/files/loaders/base.py @@ -1,4 +1,5 @@ import logging +import numpy as np from abc import ABC, abstractmethod from PIL import Image # type: ignore @@ -104,7 +105,22 @@ def get_width(self) -> int: ... +Spectrogram = np.ndarray + + class SoundFileLoader(FileLoaderBase): @abstractmethod def __len__(self) -> int: """Length of file in ms""" + + @abstractmethod + def get_spectrogram( + self, + start_time: int, + stop_time: int, + window_length: float, + max_frequency: float, + dynamic_range: float, + n_slices: int, + ) -> Optional[Spectrogram]: + ... diff --git a/ultratrace2/model/files/loaders/pydub.py b/ultratrace2/model/files/loaders/pydub.py index 470878c..3c9a851 100644 --- a/ultratrace2/model/files/loaders/pydub.py +++ b/ultratrace2/model/files/loaders/pydub.py @@ -1,6 +1,13 @@ +from typing import Optional + +import logging +import numpy as np import pydub # type: ignore -from .base import FileLoadError, SoundFileLoader +from .base import FileLoadError, SoundFileLoader, Spectrogram + + +logger = logging.getLogger(__name__) class PydubLoader(SoundFileLoader): @@ -24,3 +31,48 @@ def from_file(cls, path: str) -> "PydubLoader": return PydubLoader(path, audio_segment) except Exception as e: raise FileLoadError(f"Invalid AudioSegment ({path}), unable to read") from e + + def get_spectrogram( + self, + start_time: int, + stop_time: int, + window_length: float, # 0.005, + max_frequency: float, # 5000, + dynamic_range: float, # 90, + n_slices: int, # 10 ** 7, + ) -> Optional[Spectrogram]: + + try: + import parselmouth # type: ignore + except ImportError as e: + logger.warning(e) + return None + + start_time = max(0, start_time) + stop_time = min(len(self), stop_time) + visible_duration = stop_time - start_time + + time_step = visible_duration / n_slices + + sound_clip = parselmouth.Sound(self.get_path()).extract_part( + from_time=start_time, to_time=stop_time + ) + + spec = sound_clip.to_spectrogram( + window_length=window_length, + maximum_frequency=max_frequency, + time_step=time_step, + ) + + spec_arr = 10 * np.log10(np.flipud(spec.values)) + max_magnitude = spec_arr.max() + spec_arr = ( + ( + spec_arr.clip(max_magnitude - dynamic_range, max_magnitude) + - max_magnitude + ) + * -255 + / dynamic_range + ) + + return spec_arr From 06556704ad10e7333d068ef0e3f9c73f72caed1b Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Wed, 1 Jan 2020 09:21:26 -0800 Subject: [PATCH 2/6] SoundFileLoader: Flush out spectrogram functionality, add tests --- ultratrace2/model/files/loaders/base.py | 15 +++++-- ultratrace2/model/files/loaders/pydub.py | 41 +++++++++++------ .../files/loaders/tests/test_spectrogram.py | 44 +++++++++++++++++++ 3 files changed, 82 insertions(+), 18 deletions(-) create mode 100644 ultratrace2/model/files/loaders/tests/test_spectrogram.py diff --git a/ultratrace2/model/files/loaders/base.py b/ultratrace2/model/files/loaders/base.py index 8305748..1d10a04 100644 --- a/ultratrace2/model/files/loaders/base.py +++ b/ultratrace2/model/files/loaders/base.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod from PIL import Image # type: ignore -from typing import Sequence, Tuple, Type, TypeVar +from typing import Optional, Sequence, Tuple, Type, TypeVar from typing_extensions import Protocol @@ -116,11 +116,18 @@ def __len__(self) -> int: @abstractmethod def get_spectrogram( self, - start_time: int, - stop_time: int, + start_time_ms: int, + stop_time_ms: int, window_length: float, max_frequency: float, dynamic_range: float, n_slices: int, ) -> Optional[Spectrogram]: - ... + """Return a numpy array representing the pixels in a spectrogram. + + Ultimately, we should probably cache this to disk somehow, either + by converting it directly into a PNG via PIL or saving the raw array, + since the FFT operations are comparatively slow. + + For now, at least until we have a stable API, it makes sense to me + to just generate this on-the-fly.""" diff --git a/ultratrace2/model/files/loaders/pydub.py b/ultratrace2/model/files/loaders/pydub.py index 3c9a851..e8bd536 100644 --- a/ultratrace2/model/files/loaders/pydub.py +++ b/ultratrace2/model/files/loaders/pydub.py @@ -3,6 +3,7 @@ import logging import numpy as np import pydub # type: ignore +import warnings from .base import FileLoadError, SoundFileLoader, Spectrogram @@ -34,8 +35,8 @@ def from_file(cls, path: str) -> "PydubLoader": def get_spectrogram( self, - start_time: int, - stop_time: int, + start_time_ms: int, + stop_time_ms: int, window_length: float, # 0.005, max_frequency: float, # 5000, dynamic_range: float, # 90, @@ -48,23 +49,35 @@ def get_spectrogram( logger.warning(e) return None - start_time = max(0, start_time) - stop_time = min(len(self), stop_time) - visible_duration = stop_time - start_time + start_time_sec = min(max(0, start_time_ms), len(self)) / 1000 + stop_time_sec = max(0, min(len(self), stop_time_ms)) / 1000 + visible_duration_sec = stop_time_sec - start_time_sec - time_step = visible_duration / n_slices + time_step = visible_duration_sec / n_slices - sound_clip = parselmouth.Sound(self.get_path()).extract_part( - from_time=start_time, to_time=stop_time + spec = ( + parselmouth.Sound(self.get_path()) + .extract_part(from_time=start_time_sec, to_time=stop_time_sec) + .to_spectrogram( + window_length=window_length, + maximum_frequency=max_frequency, + time_step=time_step, + ) ) - spec = sound_clip.to_spectrogram( - window_length=window_length, - maximum_frequency=max_frequency, - time_step=time_step, - ) + with warnings.catch_warnings(): + # We filter our numpy's warnings about division by zero, since we expect + # there to be some "padding" at the start and end of the recording with + # decibel-level zero. Also, numpy does the sensible thing of defining + # `np.log10(0) := float("-inf")`, so it's not an issue for us. + warnings.filterwarnings( + "ignore", + message="divide by zero encountered in log10", + category=RuntimeWarning, + ) + spec_arr = 10 * np.log10(np.flipud(spec.values)) - spec_arr = 10 * np.log10(np.flipud(spec.values)) + # Normalize output array max_magnitude = spec_arr.max() spec_arr = ( ( diff --git a/ultratrace2/model/files/loaders/tests/test_spectrogram.py b/ultratrace2/model/files/loaders/tests/test_spectrogram.py new file mode 100644 index 0000000..f25625b --- /dev/null +++ b/ultratrace2/model/files/loaders/tests/test_spectrogram.py @@ -0,0 +1,44 @@ +import os +import PIL # type: ignore +import pytest + +from ..base import SoundFileLoader +from ..mp3 import MP3Loader + + +@pytest.mark.parametrize( + "loader, path", + [ + (MP3Loader, "./test-data/example-audio/cello82.mp3"), + (MP3Loader, "./test-data/example-audio/harpsi-cs.mp3"), + (MP3Loader, "./test-data/example-audio/gtr-nylon22.mp3"), + (MP3Loader, "./test-data/example-audio/pno-cs.mp3"), + (MP3Loader, "./test-data/example-audio/bachfugue.mp3"), + (MP3Loader, "./test-data/example-bundles/ex016/link00.mp3"), + (MP3Loader, "./test-data/example-bundles/ex003/file01.mp3"), + (MP3Loader, "./test-data/example-bundles/ex004/file00.mp3"), + (MP3Loader, "./test-data/example-bundles/ex002/file00.mp3"), + (MP3Loader, "./test-data/example-bundles/ex015/file00.mp3"), + ( + MP3Loader, + "./test-data/example-bundles/ex012/sub01/sub00/sub00/sub00/file00.mp3", + ), + (MP3Loader, "./test-data/example-bundles/ex009/file00.mp3"), + (MP3Loader, "./test-data/example-bundles/ex008/file01.mp3"), + (MP3Loader, "./test-data/example-bundles/ex008/file00.mp3"), + (MP3Loader, "./test-data/example-bundles/ex008/file02.mp3"), + ], +) +def test_interpreting_as_image(loader: SoundFileLoader, path: str) -> None: + loaded_file = loader.from_file(path) + spec_arr = loaded_file.get_spectrogram( + start_time_ms=0, + stop_time_ms=len(loaded_file), + window_length=0.005, + max_frequency=5000, + dynamic_range=90, + n_slices=10 ** 7, + ) + assert spec_arr is not None + im = PIL.Image.fromarray(spec_arr).convert("RGB").resize((800, 600)) + im.save("/tmp/" + os.path.basename(path) + ".png") From f3d9d5342f56e64ed22e7b9e58d9685b9bcfde1f Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Mon, 6 Jan 2020 16:42:28 -0800 Subject: [PATCH 3/6] FileBundle: Make access via getters instead of props This way, we can lazy-load / compute values that we don't already have at access time. --- ultratrace2/model/files/bundle.py | 23 +++++++++++++++---- ...ram.py => test_parselmouth_spectrogram.py} | 0 2 files changed, 19 insertions(+), 4 deletions(-) rename ultratrace2/model/files/loaders/tests/{test_spectrogram.py => test_parselmouth_spectrogram.py} (100%) diff --git a/ultratrace2/model/files/bundle.py b/ultratrace2/model/files/bundle.py index 7bf73ae..2faecd1 100644 --- a/ultratrace2/model/files/bundle.py +++ b/ultratrace2/model/files/bundle.py @@ -35,23 +35,38 @@ def has_impl(self) -> bool: for f in [self.alignment_file, self.image_set_file, self.sound_file] ) + def get_alignment_file(self) -> Optional[AlignmentFileLoader]: + return self.alignment_file + def set_alignment_file(self, alignment_file: AlignmentFileLoader) -> None: if self.alignment_file is not None: logger.warning("Overwriting existing alignment file") self.alignment_file = alignment_file + def get_image_set_file(self) -> Optional[ImageSetFileLoader]: + return self.image_set_file + def set_image_set_file(self, image_set_file: ImageSetFileLoader) -> None: if self.image_set_file is not None: logger.warning("Overwriting existing image-set file") self.image_set_file = image_set_file + def get_sound_file(self) -> Optional[SoundFileLoader]: + return self.sound_file + def set_sound_file(self, sound_file: SoundFileLoader) -> None: if self.sound_file is not None: logger.warning("Overwriting existing sound file") self.sound_file = sound_file def __repr__(self): - return f'Bundle("{self.name}",{self.alignment_file},{self.image_set_file},{self.sound_file})' + return ( + f'Bundle("{self.name}"' + f",{self.get_alignment_file()}" + f",{self.get_image_set_file()}" + f",{self.get_sound_file()}" + f")" + ) class FileBundleList: @@ -76,9 +91,9 @@ def __init__(self, bundles: Mapping[str, FileBundle]): self.has_sound_impl: bool = False for bundle in bundles.values(): - self.has_alignment_impl |= bundle.alignment_file is not None - self.has_image_set_impl |= bundle.image_set_file is not None - self.has_sound_impl |= bundle.sound_file is not None + self.has_alignment_impl |= bundle.get_alignment_file() is not None + self.has_image_set_impl |= bundle.get_image_set_file() is not None + self.has_sound_impl |= bundle.get_sound_file() is not None @classmethod def build_from_dir( diff --git a/ultratrace2/model/files/loaders/tests/test_spectrogram.py b/ultratrace2/model/files/loaders/tests/test_parselmouth_spectrogram.py similarity index 100% rename from ultratrace2/model/files/loaders/tests/test_spectrogram.py rename to ultratrace2/model/files/loaders/tests/test_parselmouth_spectrogram.py From 60e9f94a08c3de26c7e88bdfb59632a7291edd14 Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Mon, 6 Jan 2020 19:03:40 -0800 Subject: [PATCH 4/6] ParselmouthLoader: Add new FileLoaderBase type & implementation --- ultratrace2/model/files/__init__.py | 9 +++ ultratrace2/model/files/bundle.py | 21 ++++++- ultratrace2/model/files/loaders/__init__.py | 3 + ultratrace2/model/files/loaders/base.py | 26 ++++---- ultratrace2/model/files/loaders/pydub.py | 63 +------------------ .../tests/test_parselmouth_spectrogram.py | 50 ++++++++++++--- ultratrace2/model/files/tests/test_bundle.py | 2 +- 7 files changed, 87 insertions(+), 87 deletions(-) diff --git a/ultratrace2/model/files/__init__.py b/ultratrace2/model/files/__init__.py index bf613f7..31ef39b 100644 --- a/ultratrace2/model/files/__init__.py +++ b/ultratrace2/model/files/__init__.py @@ -68,3 +68,12 @@ ) except ImportError as e: logger.warning(e) + +try: + from .loaders import ParselmouthLoader + + __register( + [".pmpkl"], ["application/octet-stream"], ParselmouthLoader, + ) +except ImportError as e: + logger.warning(e) diff --git a/ultratrace2/model/files/bundle.py b/ultratrace2/model/files/bundle.py index 2faecd1..ce15f7d 100644 --- a/ultratrace2/model/files/bundle.py +++ b/ultratrace2/model/files/bundle.py @@ -7,6 +7,7 @@ AlignmentFileLoader, ImageSetFileLoader, SoundFileLoader, + SpectrogramFileLoader, FileLoaderBase, FileLoadError, ) @@ -23,16 +24,23 @@ def __init__( alignment_file: Optional[AlignmentFileLoader] = None, image_set_file: Optional[ImageSetFileLoader] = None, sound_file: Optional[SoundFileLoader] = None, + spectrogram_file: Optional[SpectrogramFileLoader] = None, ): self.name = name self.alignment_file = alignment_file self.image_set_file = image_set_file self.sound_file = sound_file + self.spectrogram_file = spectrogram_file def has_impl(self) -> bool: return any( f is not None - for f in [self.alignment_file, self.image_set_file, self.sound_file] + for f in [ + self.alignment_file, + self.image_set_file, + self.sound_file, + self.spectrogram_file, + ] ) def get_alignment_file(self) -> Optional[AlignmentFileLoader]: @@ -59,12 +67,21 @@ def set_sound_file(self, sound_file: SoundFileLoader) -> None: logger.warning("Overwriting existing sound file") self.sound_file = sound_file + def get_spectrogram_file(self) -> Optional[SpectrogramFileLoader]: + return self.spectrogram_file + + def set_spectrogram_file(self, spectrogram_file: SpectrogramFileLoader) -> None: + if self.spectrogram_file is not None: + logger.warning("Overwriting existing spectrogram file") + self.spectrogram_file = spectrogram_file + def __repr__(self): return ( f'Bundle("{self.name}"' f",{self.get_alignment_file()}" f",{self.get_image_set_file()}" f",{self.get_sound_file()}" + f",{self.get_spectrogram_file()}" f")" ) @@ -89,11 +106,13 @@ def __init__(self, bundles: Mapping[str, FileBundle]): self.has_alignment_impl: bool = False self.has_image_set_impl: bool = False self.has_sound_impl: bool = False + self.has_spectrogram_impl: bool = False for bundle in bundles.values(): self.has_alignment_impl |= bundle.get_alignment_file() is not None self.has_image_set_impl |= bundle.get_image_set_file() is not None self.has_sound_impl |= bundle.get_sound_file() is not None + self.has_spectrogram_impl |= bundle.get_spectrogram_file() is not None @classmethod def build_from_dir( diff --git a/ultratrace2/model/files/loaders/__init__.py b/ultratrace2/model/files/loaders/__init__.py index c27a3dd..6bc4171 100644 --- a/ultratrace2/model/files/loaders/__init__.py +++ b/ultratrace2/model/files/loaders/__init__.py @@ -10,3 +10,6 @@ from .mp3 import MP3Loader # noqa: F401 from .ogg import OggLoader # noqa: F401 from .wav import WAVLoader # noqa: F401 + +# spectrogram files +from .parselmouth_spectrogram import ParselmouthLoader # noqa: F401 diff --git a/ultratrace2/model/files/loaders/base.py b/ultratrace2/model/files/loaders/base.py index 1d10a04..cc0ce44 100644 --- a/ultratrace2/model/files/loaders/base.py +++ b/ultratrace2/model/files/loaders/base.py @@ -1,9 +1,8 @@ import logging -import numpy as np from abc import ABC, abstractmethod from PIL import Image # type: ignore -from typing import Optional, Sequence, Tuple, Type, TypeVar +from typing import Sequence, Tuple, Type, TypeVar from typing_extensions import Protocol @@ -105,16 +104,20 @@ def get_width(self) -> int: ... -Spectrogram = np.ndarray - - class SoundFileLoader(FileLoaderBase): @abstractmethod def __len__(self) -> int: """Length of file in ms""" + +class SpectrogramFileLoader(FileLoaderBase): + @classmethod + @abstractmethod + def from_sound_file(cls: Type[Self], sound_file: SoundFileLoader) -> Self: + ... + @abstractmethod - def get_spectrogram( + def get_image( self, start_time_ms: int, stop_time_ms: int, @@ -122,12 +125,5 @@ def get_spectrogram( max_frequency: float, dynamic_range: float, n_slices: int, - ) -> Optional[Spectrogram]: - """Return a numpy array representing the pixels in a spectrogram. - - Ultimately, we should probably cache this to disk somehow, either - by converting it directly into a PNG via PIL or saving the raw array, - since the FFT operations are comparatively slow. - - For now, at least until we have a stable API, it makes sense to me - to just generate this on-the-fly.""" + ) -> Image.Image: + ... diff --git a/ultratrace2/model/files/loaders/pydub.py b/ultratrace2/model/files/loaders/pydub.py index e8bd536..50dba62 100644 --- a/ultratrace2/model/files/loaders/pydub.py +++ b/ultratrace2/model/files/loaders/pydub.py @@ -1,11 +1,7 @@ -from typing import Optional - import logging -import numpy as np import pydub # type: ignore -import warnings -from .base import FileLoadError, SoundFileLoader, Spectrogram +from .base import FileLoadError, SoundFileLoader logger = logging.getLogger(__name__) @@ -32,60 +28,3 @@ def from_file(cls, path: str) -> "PydubLoader": return PydubLoader(path, audio_segment) except Exception as e: raise FileLoadError(f"Invalid AudioSegment ({path}), unable to read") from e - - def get_spectrogram( - self, - start_time_ms: int, - stop_time_ms: int, - window_length: float, # 0.005, - max_frequency: float, # 5000, - dynamic_range: float, # 90, - n_slices: int, # 10 ** 7, - ) -> Optional[Spectrogram]: - - try: - import parselmouth # type: ignore - except ImportError as e: - logger.warning(e) - return None - - start_time_sec = min(max(0, start_time_ms), len(self)) / 1000 - stop_time_sec = max(0, min(len(self), stop_time_ms)) / 1000 - visible_duration_sec = stop_time_sec - start_time_sec - - time_step = visible_duration_sec / n_slices - - spec = ( - parselmouth.Sound(self.get_path()) - .extract_part(from_time=start_time_sec, to_time=stop_time_sec) - .to_spectrogram( - window_length=window_length, - maximum_frequency=max_frequency, - time_step=time_step, - ) - ) - - with warnings.catch_warnings(): - # We filter our numpy's warnings about division by zero, since we expect - # there to be some "padding" at the start and end of the recording with - # decibel-level zero. Also, numpy does the sensible thing of defining - # `np.log10(0) := float("-inf")`, so it's not an issue for us. - warnings.filterwarnings( - "ignore", - message="divide by zero encountered in log10", - category=RuntimeWarning, - ) - spec_arr = 10 * np.log10(np.flipud(spec.values)) - - # Normalize output array - max_magnitude = spec_arr.max() - spec_arr = ( - ( - spec_arr.clip(max_magnitude - dynamic_range, max_magnitude) - - max_magnitude - ) - * -255 - / dynamic_range - ) - - return spec_arr diff --git a/ultratrace2/model/files/loaders/tests/test_parselmouth_spectrogram.py b/ultratrace2/model/files/loaders/tests/test_parselmouth_spectrogram.py index f25625b..9963394 100644 --- a/ultratrace2/model/files/loaders/tests/test_parselmouth_spectrogram.py +++ b/ultratrace2/model/files/loaders/tests/test_parselmouth_spectrogram.py @@ -4,10 +4,11 @@ from ..base import SoundFileLoader from ..mp3 import MP3Loader +from ..parselmouth_spectrogram import ParselmouthLoader @pytest.mark.parametrize( - "loader, path", + "sound_file_loader, path", [ (MP3Loader, "./test-data/example-audio/cello82.mp3"), (MP3Loader, "./test-data/example-audio/harpsi-cs.mp3"), @@ -29,16 +30,49 @@ (MP3Loader, "./test-data/example-bundles/ex008/file02.mp3"), ], ) -def test_interpreting_as_image(loader: SoundFileLoader, path: str) -> None: - loaded_file = loader.from_file(path) - spec_arr = loaded_file.get_spectrogram( +def test_loading_from_sound_file(sound_file_loader: SoundFileLoader, path: str) -> None: + try: + expected_pm_path = path + ".pmpkl" + assert not os.path.exists(expected_pm_path) + sound_file = sound_file_loader.from_file(path) + pm_file = ParselmouthLoader.from_sound_file(sound_file) + assert isinstance(pm_file, ParselmouthLoader) + assert pm_file.get_path() == expected_pm_path + assert os.path.exists(expected_pm_path) + finally: + if os.path.exists(expected_pm_path): + os.remove(expected_pm_path) + + +@pytest.mark.parametrize( + "path", + [ + "./test-data/example-parselmouth/00.pmpkl", + "./test-data/example-parselmouth/01.pmpkl", + "./test-data/example-parselmouth/02.pmpkl", + "./test-data/example-parselmouth/03.pmpkl", + "./test-data/example-parselmouth/04.pmpkl", + "./test-data/example-parselmouth/05.pmpkl", + "./test-data/example-parselmouth/06.pmpkl", + "./test-data/example-parselmouth/07.pmpkl", + "./test-data/example-parselmouth/08.pmpkl", + "./test-data/example-parselmouth/09.pmpkl", + "./test-data/example-parselmouth/10.pmpkl", + "./test-data/example-parselmouth/11.pmpkl", + "./test-data/example-parselmouth/12.pmpkl", + "./test-data/example-parselmouth/13.pmpkl", + "./test-data/example-parselmouth/14.pmpkl", + ], +) +def test_loading_from_pmpkl_file(path: str) -> None: + pm_file = ParselmouthLoader.from_file(path) + assert isinstance(pm_file, ParselmouthLoader) + im = pm_file.get_image( start_time_ms=0, - stop_time_ms=len(loaded_file), + stop_time_ms=10, window_length=0.005, max_frequency=5000, dynamic_range=90, n_slices=10 ** 7, ) - assert spec_arr is not None - im = PIL.Image.fromarray(spec_arr).convert("RGB").resize((800, 600)) - im.save("/tmp/" + os.path.basename(path) + ".png") + assert isinstance(im, PIL.Image.Image) diff --git a/ultratrace2/model/files/tests/test_bundle.py b/ultratrace2/model/files/tests/test_bundle.py index fb0e558..d633473 100644 --- a/ultratrace2/model/files/tests/test_bundle.py +++ b/ultratrace2/model/files/tests/test_bundle.py @@ -19,7 +19,7 @@ def test_empty_file_bundle_constructor(kwargs) -> None: fb = FileBundle("test", **kwargs) assert not fb.has_impl() - assert str(fb) == 'Bundle("test",None,None,None)' + assert str(fb) == 'Bundle("test",None,None,None,None)' def test_build_from_nonexistent_dir(mocker) -> None: From 54beeb8e05cc3e2344c5add789b9b94834e8bde1 Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Mon, 6 Jan 2020 19:13:32 -0800 Subject: [PATCH 5/6] FileBundle: Automatically create Spectrograms for files w/ Sound This commit relies on the existence of a particular sub-type of SpectrogramLoader being present, which breaks some of our assumptions. This functionality should probably be implemented some other way ... possibly via the registry? --- ultratrace2/model/files/bundle.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ultratrace2/model/files/bundle.py b/ultratrace2/model/files/bundle.py index ce15f7d..aff4f8b 100644 --- a/ultratrace2/model/files/bundle.py +++ b/ultratrace2/model/files/bundle.py @@ -11,6 +11,7 @@ FileLoaderBase, FileLoadError, ) +from .loaders.parselmouth_spectrogram import ParselmouthLoader from .registry import get_loader_for @@ -68,6 +69,19 @@ def set_sound_file(self, sound_file: SoundFileLoader) -> None: self.sound_file = sound_file def get_spectrogram_file(self) -> Optional[SpectrogramFileLoader]: + if self.spectrogram_file is None: + sound_file = self.get_sound_file() + if sound_file is not None: + try: + logger.debug( + "No spectrogram file found, trying to create from sound file" + ) + self.set_spectrogram_file( + ParselmouthLoader.from_sound_file(sound_file) + ) + except FileLoadError as e: + logger.warning(e) + return self.spectrogram_file def set_spectrogram_file(self, spectrogram_file: SpectrogramFileLoader) -> None: From 11a9fc5883dc0599e19cc4924426465588e45794 Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Fri, 10 Jan 2020 10:35:07 -0800 Subject: [PATCH 6/6] spectrogram: Add the parselmouth loader (I think I accidentally didn't include this file in any earlier commits ... RIP) --- .../files/loaders/parselmouth_spectrogram.py | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 ultratrace2/model/files/loaders/parselmouth_spectrogram.py diff --git a/ultratrace2/model/files/loaders/parselmouth_spectrogram.py b/ultratrace2/model/files/loaders/parselmouth_spectrogram.py new file mode 100644 index 0000000..305919f --- /dev/null +++ b/ultratrace2/model/files/loaders/parselmouth_spectrogram.py @@ -0,0 +1,101 @@ +from PIL import Image # type: ignore + +import logging +import numpy as np +import parselmouth # type: ignore +import pickle +import warnings + +from .base import FileLoadError, SoundFileLoader, SpectrogramFileLoader + + +logger = logging.getLogger(__name__) + + +class ParselmouthLoader(SpectrogramFileLoader): + def get_path(self) -> str: + return self._path + + def set_path(self, path) -> None: + self._path = path + + def __init__(self, path: str, pm_sound: parselmouth.Sound): + self.set_path(path) + self.pm_sound = pm_sound + + def save(self) -> None: + with open(self.get_path(), "wb") as fp: + pickle.dump(self.pm_sound.as_array(), fp) + logger.debug(f"Wrote ParselmouthLoader to disk: {self.get_path()}") + + @classmethod + def from_file(cls, path: str) -> "ParselmouthLoader": + try: + with open(path, "rb") as fp: + pm_sound = parselmouth.Sound(pickle.load(fp)) + return cls(path, pm_sound) + except Exception as e: + raise FileLoadError from e + + load = from_file # alias + + @classmethod + def from_sound_file(cls, sound_file: SoundFileLoader) -> "ParselmouthLoader": + try: + pm_sound = parselmouth.Sound(sound_file.get_path()) + pm_path = f"{sound_file.get_path()}.pmpkl" + pm = cls(pm_path, pm_sound) + pm.save() + return pm + except Exception as e: + raise FileLoadError from e + + def get_image( + self, + start_time_ms: int, + stop_time_ms: int, + window_length: float, # 0.005, + max_frequency: float, # 5000, + dynamic_range: float, # 90, + n_slices: int, # 10 ** 7, + ) -> Image.Image: + + # NB: need to do validation of *_time_ms BEFORE calling + start_time_sec = start_time_ms / 1000 + stop_time_sec = stop_time_ms / 1000 + visible_duration_sec = stop_time_sec - start_time_sec + + time_step = visible_duration_sec / n_slices + + spec = self.pm_sound.extract_part( + from_time=start_time_sec, to_time=stop_time_sec + ).to_spectrogram( + window_length=window_length, + maximum_frequency=max_frequency, + time_step=time_step, + ) + + with warnings.catch_warnings(): + # We filter our numpy's warnings about division by zero, since we expect + # there to be some "padding" at the start and end of the recording with + # decibel-level zero. Also, numpy does the sensible thing of defining + # `np.log10(0) := float("-inf")`, so it's not an issue for us. + warnings.filterwarnings( + "ignore", + message="divide by zero encountered in log10", + category=RuntimeWarning, + ) + spec_arr = 10 * np.log10(np.flipud(spec.values)) + + # Normalize output array + max_magnitude = spec_arr.max() + spec_arr = ( + ( + spec_arr.clip(max_magnitude - dynamic_range, max_magnitude) + - max_magnitude + ) + * -255 + / dynamic_range + ) + + return Image.fromarray(spec_arr).convert("RGB")