From c285b94ad968233c9cc94756769c21f8e5200e38 Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Wed, 10 Jul 2019 22:35:53 +0200 Subject: [PATCH 01/14] first pass --- __init__.py | 1 - audfprint/__init__.py | 5 + audfprint.py => audfprint/audfprint.py | 31 +++--- .../audfprint_analyze.py | 25 +++-- .../audfprint_match.py | 21 +++-- audio_read.py => audfprint/audio_read.py | 0 hash_table.py => audfprint/hash_table.py | 27 ++++-- audfprint/stft.py | 93 ++++++++++++++++++ stft.py | 94 ------------------- 9 files changed, 164 insertions(+), 133 deletions(-) delete mode 100644 __init__.py create mode 100644 audfprint/__init__.py rename audfprint.py => audfprint/audfprint.py (94%) mode change 100755 => 100644 rename audfprint_analyze.py => audfprint/audfprint_analyze.py (96%) rename audfprint_match.py => audfprint/audfprint_match.py (96%) rename audio_read.py => audfprint/audio_read.py (100%) rename hash_table.py => audfprint/hash_table.py (93%) create mode 100644 audfprint/stft.py delete mode 100644 stft.py diff --git a/__init__.py b/__init__.py deleted file mode 100644 index 9bad579..0000000 --- a/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# coding=utf-8 diff --git a/audfprint/__init__.py b/audfprint/__init__.py new file mode 100644 index 0000000..66fd128 --- /dev/null +++ b/audfprint/__init__.py @@ -0,0 +1,5 @@ +# coding=utf-8 +import logging + +LOG = logging.getLogger(__name__) +LOG.addHandler(logging.NullHandler()) diff --git a/audfprint.py b/audfprint/audfprint.py old mode 100755 new mode 100644 similarity index 94% rename from audfprint.py rename to audfprint/audfprint.py index f5bb51b..7999075 --- a/audfprint.py +++ b/audfprint/audfprint.py @@ -21,6 +21,8 @@ import multiprocessing import joblib +from audfprint import LOG + # The actual analyzer class/code import audfprint_analyze # Access to match functions, used in command line interface @@ -119,7 +121,8 @@ def file_precompute_peaks_or_hashes(analyzer, filename, precompdir, def file_precompute(analyzer, filename, precompdir, type='peaks', skip_existing=False, strip_prefix=None): """ Perform precompute action for one file, return list of message strings """ - print(time.ctime(), "precomputing", type, "for", filename, "...") + LOG.info('%s precomputing %s for %s ...', time.ctime(), type, filename) + #print(time.ctime(), "precomputing", type, "for", filename, "...") hashes_not_peaks = (type == 'hashes') return file_precompute_peaks_or_hashes(analyzer, filename, precompdir, hashes_not_peaks=hashes_not_peaks, @@ -406,7 +409,7 @@ def main(argv): cmd = cmdlist[0] # Setup output function - report = setup_reporter(args) + report = setup_reporter(args) # FFS # Keep track of wall time initticks = time_clock() @@ -435,10 +438,9 @@ def main(argv): # Check that the output directory can be created before we start ensure_dir(os.path.split(dbasename)[0]) # Create a new hash table - hash_tab = hash_table.HashTable( - hashbits=int(args['--hashbits']), - depth=int(args['--bucketsize']), - maxtime=(1 << int(args['--maxtimebits']))) + hash_tab = hash_table.HashTable(hashbits=int(args['--hashbits']), + depth=int(args['--bucketsize']), + maxtime=(1 << int(args['--maxtimebits']))) # Set its samplerate param if analyzer: hash_tab.params['samplerate'] = analyzer.target_sr @@ -446,12 +448,14 @@ def main(argv): else: # Load existing hash table file (add, match, merge) if args['--verbose']: - report([time.ctime() + " Reading hash table " + dbasename]) + LOG.debug('Reading hashtable %s', dbasename) + # report([time.ctime() + " Reading hash table " + dbasename]) hash_tab = hash_table.HashTable(dbasename) if analyzer and 'samplerate' in hash_tab.params \ and hash_tab.params['samplerate'] != analyzer.target_sr: # analyzer.target_sr = hash_tab.params['samplerate'] - print("db samplerate overridden to ", analyzer.target_sr) + LOG.info('db samplerate overridden to %s', analyzer.target_sr) + # print("db samplerate overridden to ", analyzer.target_sr) else: # The command IS precompute # dummy empty hash table @@ -488,10 +492,13 @@ def main(argv): elapsedtime = time_clock() - initticks if analyzer and analyzer.soundfiletotaldur > 0.: - print("Processed " - + "%d files (%.1f s total dur) in %.1f s sec = %.3f x RT" \ - % (analyzer.soundfilecount, analyzer.soundfiletotaldur, - elapsedtime, (elapsedtime / analyzer.soundfiletotaldur))) + LOG.debug('Processed %d files %.1f s total dur in %.1f s sec = %.3f x RT', + analyzer.soundfilecount, analyzer.soundfiletotaldur, + elapsedtime, (elapsedtime / analyzer.soundfiletotaldur)) + # print("Processed " + # + "%d files (%.1f s total dur) in %.1f s sec = %.3f x RT" \ + # % (analyzer.soundfilecount, analyzer.soundfiletotaldur, + # elapsedtime, (elapsedtime / analyzer.soundfiletotaldur))) # Save the hash table file if it has been modified if hash_tab and hash_tab.dirty: diff --git a/audfprint_analyze.py b/audfprint/audfprint_analyze.py similarity index 96% rename from audfprint_analyze.py rename to audfprint/audfprint_analyze.py index 3a73ec3..6dc253b 100644 --- a/audfprint_analyze.py +++ b/audfprint/audfprint_analyze.py @@ -24,6 +24,7 @@ import audio_read # For utility, glob2hashtable import hash_table +from audfprint import LOG import stft # ############### Globals ############### # @@ -86,7 +87,7 @@ def landmarks2hashes(landmarks): landmarks = np.array(landmarks) # Deal with special case of empty landmarks. if landmarks.shape[0] == 0: - return np.zeros((0, 2), dtype=np.int32) + return np.zeros((0, 2), dtype=np.int32) hashes = np.zeros((landmarks.shape[0], 2), dtype=np.int32) hashes[:, 0] = landmarks[:, 0] hashes[:, 1] = (((landmarks[:, 1] & B1_MASK) << B1_SHIFT) @@ -287,7 +288,8 @@ def find_peaks(self, d, sr): else: # The sgram is identically zero, i.e., the input signal was identically # zero. Not good, but let's let it through for now. - print("find_peaks: Warning: input signal is identically zero.") + LOG.debug('find_peaks: Warning: input signal is identically zero.') + # print("find_peaks: Warning: input signal is identically zero.") # High-pass filter onset emphasis # [:-1,] discards top bin (nyquist) of sgram so bins fit in 8 bits sgram = np.array([scipy.signal.lfilter([1, -1], @@ -329,7 +331,7 @@ def peaks2landmarks(self, pklist): for peak in peaks_at[col]: pairsthispeak = 0 for col2 in range(col + self.mindt, - min(scols, col + self.targetdt)): + min(scols, col + self.targetdt)): if pairsthispeak < self.maxpairsperpeak: for peak2 in peaks_at[col2]: if abs(peak2 - peak) < self.targetdf: @@ -357,11 +359,13 @@ def wavfile2peaks(self, filename, shifts=None): # [d, sr] = librosa.load(filename, sr=self.target_sr) d, sr = audio_read.audio_read(filename, sr=self.target_sr, channels=1) except Exception as e: # audioread.NoBackendError: - message = "wavfile2peaks: Error reading " + filename + message = "wavfile2peaks: Error reading %s" % filename if self.fail_on_error: - print(e) + # print(e) + LOG.error('%s', message) raise IOError(message) - print(message, "skipping") + LOG.debug('%s skipping', message) + # print(message, "skipping") d = [] sr = self.target_sr # Store duration in a global because it's hard to handle @@ -569,13 +573,16 @@ def glob2hashtable(pattern, density=20.0): totdur = 0.0 tothashes = 0 for ix, file_ in enumerate(filelist): - print(time.ctime(), "ingesting #", ix, ":", file_, "...") + LOG.debug('Ingesting #%s: %s ...', ix, file_) + # print(time.ctime(), "ingesting #", ix, ":", file_, "...") dur, nhash = g2h_analyzer.ingest(ht, file_) totdur += dur tothashes += nhash elapsedtime = time.clock() - initticks - print("Added", tothashes, "(", tothashes / totdur, "hashes/sec) at ", - elapsedtime / totdur, "x RT") + LOG.debug('Added %s %s hashes/sec at %s x RT', + tothashes, (tothashes / totdur), (elapsedtime / totdur)) + # print("Added", tothashes, "(", tothashes / totdur, "hashes/sec) at ", + # elapsedtime / totdur, "x RT") return ht diff --git a/audfprint_match.py b/audfprint/audfprint_match.py similarity index 96% rename from audfprint_match.py rename to audfprint/audfprint_match.py index d97026e..29c0d64 100644 --- a/audfprint_match.py +++ b/audfprint/audfprint_match.py @@ -21,6 +21,8 @@ except: pass +from audfprint import LOG + import audfprint_analyze import audio_read import stft @@ -40,7 +42,8 @@ def process_info(): def log(message): """ log info with stats """ - print('%s physmem=%s utime=%s %s' % (time.ctime(), process_info())) + LOG.debug('physmem=%s utime=%s %s' % process_info(), message) + # print('%s physmem=%s utime=%s %s' % (time.ctime(), process_info())) def encpowerof2(val): @@ -368,9 +371,11 @@ def match_file(self, analyzer, ht, filename, number=None): numberstring = "#%d" % number else: numberstring = "" - print(time.ctime(), "Analyzed", numberstring, filename, "of", - ('%.3f' % durd), "s " - "to", len(q_hashes), "hashes") + LOG.debug('Analyzed %s %s of %.3f s to %s hashes', + numberstring, filename, durd, len(q_hashes)) + # print(time.ctime(), "Analyzed", numberstring, filename, "of", + # ('%.3f' % durd), "s " + # "to", len(q_hashes), "hashes") # Run query rslts = self.match_hashes(ht, q_hashes) # Post filtering @@ -378,6 +383,7 @@ def match_file(self, analyzer, ht, filename, number=None): rslts = rslts[(-rslts[:, 2]).argsort(), :] return rslts[:self.max_returns, :], durd, len(q_hashes) + # WTF should we do about this. def file_match_to_msgs(self, analyzer, ht, qry, number=None): """ Perform a match on a single input file, return list of message strings """ @@ -459,10 +465,11 @@ def illustrate_match(self, analyzer, ht, filename): np.array([[x[1], x[2]] for x in mlms]).T, '.-r') # Add title - plt.title(filename + " : Matched as " + ht.names[results[0][0]] - + (" with %d of %d hashes" % (len(matchhashes), - len(q_hashes)))) + title = '%s :Matched as %s (with %d of %d hashes' % ( + ht.names[results[0][0]], len(matchhashes), len(q_hashes)) + # Display + plt.title(title) plt.show() # Return return results diff --git a/audio_read.py b/audfprint/audio_read.py similarity index 100% rename from audio_read.py rename to audfprint/audio_read.py diff --git a/hash_table.py b/audfprint/hash_table.py similarity index 93% rename from hash_table.py rename to audfprint/hash_table.py index 1f853c7..28a59e6 100644 --- a/hash_table.py +++ b/audfprint/hash_table.py @@ -28,6 +28,7 @@ import cPickle as pickle # Py2 pickle_options = {} +from audfprint import LOG # Current format version HT_VERSION = 20170724 @@ -192,9 +193,11 @@ def save(self, name, params=None, file_object=None): nhashes = sum(self.counts) # Report the proportion of dropped hashes (overfull table) dropped = nhashes - sum(np.minimum(self.depth, self.counts)) - print("Saved fprints for", sum(n is not None for n in self.names), - "files (", nhashes, "hashes) to", name, - "(%.2f%% dropped)" % (100.0 * dropped / max(1, nhashes))) + LOG.debug('Saved fprint for %s files (%s hashes) to %s (%.2f%% dropped)', + sum(n is not None for n in self.names), nhashes, name, (100.0 * dropped / max(1, nhashes))) + # print("Saved fprints for", sum(n is not None for n in self.names), + # "files (", nhashes, "hashes) to", name, + # "(%.2f%% dropped)" % (100.0 * dropped / max(1, nhashes))) def load(self, name): """ Read either pklz or mat-format hash table file """ @@ -206,9 +209,11 @@ def load(self, name): nhashes = sum(self.counts) # Report the proportion of dropped hashes (overfull table) dropped = nhashes - sum(np.minimum(self.depth, self.counts)) - print("Read fprints for", sum(n is not None for n in self.names), - "files (", nhashes, "hashes) from", name, - "(%.2f%% dropped)" % (100.0 * dropped / max(1, nhashes))) + LOG.debug('Read fprint for %s files (%s hashes) from %s (%.2f%% dropped)', + sum(n is not None for n in self.names), nhashes, name, (100.0 * dropped / max(1, nhashes))) + # print("Read fprints for", sum(n is not None for n in self.names), + # "files (", nhashes, "hashes) from", name, + # "(%.2f%% dropped)" % (100.0 * dropped / max(1, nhashes))) def load_pkl(self, name, file_object=None): """ Read hash table values from pickle file . """ @@ -361,7 +366,8 @@ def remove(self, name): self.names[id_] = None self.hashesperid[id_] = 0 self.dirty = True - print("Removed", name, "(", hashes_removed, "hashes).") + LOG.debug('Removed %s (%s) hashes.', name, hashes_removed) + # print("Removed", name, "(", hashes_removed, "hashes).") def retrieve(self, name): """Return an np.array of (time, hash) pairs found in the table.""" @@ -384,8 +390,9 @@ def retrieve(self, name): def list(self, print_fn=None): """ List all the known items. """ - if not print_fn: - print_fn = print + # print_fn is not in use but keeping it for compat. + for name, count in zip(self.names, self.hashesperid): if name: - print_fn(name + " (" + str(count) + " hashes)") + LOG.info('%s (%s) hashes', name, count) + #print_fn(name + " (" + str(count) + " hashes)") diff --git a/audfprint/stft.py b/audfprint/stft.py new file mode 100644 index 0000000..bc29fc9 --- /dev/null +++ b/audfprint/stft.py @@ -0,0 +1,93 @@ +"""Provide stft to avoid librosa dependency. + +This implementation is based on routines from +https://github.com/tensorflow/models/blob/master/research/audioset/mel_features.py +""" + +from __future__ import division + +import numpy as np + + +def frame(data, window_length, hop_length): + """Convert array into a sequence of successive possibly overlapping frames. + + An n-dimensional array of shape (num_samples, ...) is converted into an + (n+1)-D array of shape (num_frames, window_length, ...), where each frame + starts hop_length points after the preceding one. + + This is accomplished using stride_tricks, so the original data is not + copied. However, there is no zero-padding, so any incomplete frames at the + end are not included. + + Args: + data: np.array of dimension N >= 1. + window_length: Number of samples in each frame. + hop_length: Advance (in samples) between each window. + + Returns: + (N+1)-D np.array with as many rows as there are complete frames that can be + extracted. + """ + num_samples = data.shape[0] + num_frames = 1 + ((num_samples - window_length) // hop_length) + shape = (num_frames, window_length) + data.shape[1:] + strides = (data.strides[0] * hop_length,) + data.strides + return np.lib.stride_tricks.as_strided(data, shape=shape, strides=strides) + + +def periodic_hann(window_length): + """Calculate a "periodic" Hann window. + + The classic Hann window is defined as a raised cosine that starts and + ends on zero, and where every value appears twice, except the middle + point for an odd-length window. Matlab calls this a "symmetric" window + and np.hanning() returns it. However, for Fourier analysis, this + actually represents just over one cycle of a period N-1 cosine, and + thus is not compactly expressed on a length-N Fourier basis. Instead, + it's better to use a raised cosine that ends just before the final + zero value - i.e. a complete cycle of a period-N cosine. Matlab + calls this a "periodic" window. This routine calculates it. + + Args: + window_length: The number of points in the returned window. + + Returns: + A 1D np.array containing the periodic hann window. + """ + return 0.5 - (0.5 * np.cos(2 * np.pi / window_length * np.arange(window_length))) + + +def stft(signal, n_fft, hop_length=None, window=None): + """Calculate the short-time Fourier transform. + + Args: + signal: 1D np.array of the input time-domain signal. + n_fft: Size of the FFT to apply. + hop_length: Advance (in samples) between each frame passed to FFT. Defaults + to half the window length. + window: Length of each block of samples to pass to FFT, or vector of window + values. Defaults to n_fft. + + Returns: + 2D np.array where each column contains the complex values of the + fft_length/2+1 unique values of the FFT for the corresponding frame of + input samples ("spectrogram transposition"). + """ + if window is None: + window = n_fft + if isinstance(window, (int, float)): + # window holds the window length, need to make the actual window. + window = periodic_hann(int(window)) + window_length = len(window) + if not hop_length: + hop_length = window_length // 2 + # Default librosa STFT behavior. + pad_mode = "reflect" + signal = np.pad(signal, (n_fft // 2), mode=pad_mode) + frames = frame(signal, window_length, hop_length) + # Apply frame window to each frame. We use a periodic Hann (cosine of period + # window_length) instead of the symmetric Hann of np.hanning (period + # window_length-1). + windowed_frames = frames * window + return np.fft.rfft(windowed_frames, n_fft).transpose() diff --git a/stft.py b/stft.py deleted file mode 100644 index 6dcd772..0000000 --- a/stft.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Provide stft to avoid librosa dependency. - -This implementation is based on routines from -https://github.com/tensorflow/models/blob/master/research/audioset/mel_features.py -""" - -from __future__ import division - -import numpy as np - - -def frame(data, window_length, hop_length): - """Convert array into a sequence of successive possibly overlapping frames. - - An n-dimensional array of shape (num_samples, ...) is converted into an - (n+1)-D array of shape (num_frames, window_length, ...), where each frame - starts hop_length points after the preceding one. - - This is accomplished using stride_tricks, so the original data is not - copied. However, there is no zero-padding, so any incomplete frames at the - end are not included. - - Args: - data: np.array of dimension N >= 1. - window_length: Number of samples in each frame. - hop_length: Advance (in samples) between each window. - - Returns: - (N+1)-D np.array with as many rows as there are complete frames that can be - extracted. - """ - num_samples = data.shape[0] - num_frames = 1 + ((num_samples - window_length) // hop_length) - shape = (num_frames, window_length) + data.shape[1:] - strides = (data.strides[0] * hop_length,) + data.strides - return np.lib.stride_tricks.as_strided(data, shape=shape, strides=strides) - - -def periodic_hann(window_length): - """Calculate a "periodic" Hann window. - - The classic Hann window is defined as a raised cosine that starts and - ends on zero, and where every value appears twice, except the middle - point for an odd-length window. Matlab calls this a "symmetric" window - and np.hanning() returns it. However, for Fourier analysis, this - actually represents just over one cycle of a period N-1 cosine, and - thus is not compactly expressed on a length-N Fourier basis. Instead, - it's better to use a raised cosine that ends just before the final - zero value - i.e. a complete cycle of a period-N cosine. Matlab - calls this a "periodic" window. This routine calculates it. - - Args: - window_length: The number of points in the returned window. - - Returns: - A 1D np.array containing the periodic hann window. - """ - return 0.5 - (0.5 * np.cos(2 * np.pi / window_length * - np.arange(window_length))) - - -def stft(signal, n_fft, hop_length=None, window=None): - """Calculate the short-time Fourier transform. - - Args: - signal: 1D np.array of the input time-domain signal. - n_fft: Size of the FFT to apply. - hop_length: Advance (in samples) between each frame passed to FFT. Defaults - to half the window length. - window: Length of each block of samples to pass to FFT, or vector of window - values. Defaults to n_fft. - - Returns: - 2D np.array where each column contains the complex values of the - fft_length/2+1 unique values of the FFT for the corresponding frame of - input samples ("spectrogram transposition"). - """ - if window is None: - window = n_fft - if isinstance(window, (int, float)): - # window holds the window length, need to make the actual window. - window = periodic_hann(int(window)) - window_length = len(window) - if not hop_length: - hop_length = window_length // 2 - # Default librosa STFT behavior. - pad_mode = 'reflect' - signal = np.pad(signal, (n_fft // 2), mode=pad_mode) - frames = frame(signal, window_length, hop_length) - # Apply frame window to each frame. We use a periodic Hann (cosine of period - # window_length) instead of the symmetric Hann of np.hanning (period - # window_length-1). - windowed_frames = frames * window - return np.fft.rfft(windowed_frames, n_fft).transpose() From 800b68ed9389d7d46abb87db36c42e49e248ecc1 Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Wed, 10 Jul 2019 22:45:14 +0200 Subject: [PATCH 02/14] add setup.py file --- setup.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..945c489 --- /dev/null +++ b/setup.py @@ -0,0 +1,95 @@ +# coding=UTF-8 + +""" +https://packaging.python.org/en/latest/distributing.html +https://github.com/pypa/sampleproject +""" + +from setuptools import setup, find_packages + +# Possibly convert the README.md to .rst-format +try: + import pypandoc + README = pypandoc.convert('README.md', 'rst') +except ImportError: + print("warning: pypandoc module not found, could not convert Markdown to RST") + README = open('README.md', 'r').read() + + +f = open('requirements.txt', 'r').readlines() +REQ = [] +for line in f: + if line and not line.startswith('#'): + REQ.append(line) + + +setup( + name='audfprint', + + # Version number is automatically extracted from Git + # https://pypi.python.org/pypi/setuptools_scm + # https://packaging.python.org/en/latest/single_source_version.html + use_scm_version=True, + setup_requires=['setuptools_scm', 'pypandoc'], + # version='0.0.1', + + description='Landmark-based audio fingerprinting.', + long_description=README, + + # The project's main homepage. + url='https://github.com/dpwe/audfprint', + + # Author details + author='Dan Ellis', + author_email='some@email.com', + + # Choose your license + license='GPL3', + + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + 'Development Status :: 4 - Beta', + + # Indicate who your project is intended for + 'Intended Audience :: End Users/Desktop', + 'Environment :: Console', + 'Topic :: Multimedia :: Audio', + + # Pick your license as you wish (should match "license" above) + #'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + + # Specify the Python versions you support here. + 'Programming Language :: Python :: 2.7, 3.6', + ], + + # What does your project relate to? + keywords='landmark audioprint, fingerprint', + + # You can just specify the packages manually here if your project is + # simple. Or you can use find_packages(). + packages=find_packages(exclude=['tests']), + include_package_data=True, + + + # List run-time dependencies here. These will be installed by pip when + # your project is installed. For an analysis of "install_requires" vs pip's + # requirements files see: + # https://packaging.python.org/en/latest/requirements.html + install_requires=REQ, + + # List additional groups of dependencies here (e.g. development + # dependencies). You can install these using the following syntax, + # for example: + # $ pip install -e .[dev,test] + + # To provide executable scripts, use entry points in preference to the + # "scripts" keyword. Entry points provide cross-platform support and allow + # pip to create the appropriate form of executable for the target platform. + entry_points={ + 'console_scripts': [ + 'audfprint=audfprint.audfprint:main', + ] + }, + + +) From 360d7896a0777b3a73601d9a969e99a9fc013e17 Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Wed, 10 Jul 2019 23:21:46 +0200 Subject: [PATCH 03/14] Add some file handler and change some imports. --- .gitignore | 1 + audfprint/audfprint.py | 27 ++++++++++------ audfprint/audfprint_analyze.py | 9 +++--- audfprint/audfprint_match.py | 6 ++-- tests/profile_audfprint.py | 28 ----------------- tests/test_profile_audfprint.py | 31 +++++++++++++++++++ ...tch.py => test_profile_audfprint_match.py} | 0 tests/{report_prof.py => test_report_prof.py} | 0 8 files changed, 58 insertions(+), 44 deletions(-) delete mode 100644 tests/profile_audfprint.py create mode 100644 tests/test_profile_audfprint.py rename tests/{profile_audfprint_match.py => test_profile_audfprint_match.py} (100%) rename tests/{report_prof.py => test_report_prof.py} (100%) diff --git a/.gitignore b/.gitignore index 5fd1337..08db47a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ var/ *.egg-info/ .installed.cfg *.egg +*.eggs # Installer logs pip-log.txt diff --git a/audfprint/audfprint.py b/audfprint/audfprint.py index 7999075..f4735f1 100644 --- a/audfprint/audfprint.py +++ b/audfprint/audfprint.py @@ -9,6 +9,7 @@ 2014-05-25 Dan Ellis dpwe@ee.columbia.edu """ from __future__ import division, print_function +from logging.handlers import RotatingFileHandler # For reporting progress time import time @@ -24,11 +25,11 @@ from audfprint import LOG # The actual analyzer class/code -import audfprint_analyze +from audfprint import audfprint_analyze # Access to match functions, used in command line interface -import audfprint_match +from audfprint import audfprint_match # My hash_table implementation -import hash_table +from audfprint import hash_table if sys.version_info[0] >= 3: @@ -323,19 +324,26 @@ def setup_matcher(args): # Command to construct the reporter object def setup_reporter(args): """ Creates a logging function, either to stderr or file""" + # not really sure how we should handle this. + # but lets do this for now. + opfile = args['--opfile'] if opfile and len(opfile): - f = open(opfile, "w") + # Add a file handle to the msges is written to the log. + rfh = RotatingFileHandler(opfile, 'a', 512000, 3) + LOG.addHandler(rfh) def report(msglist): """Log messages to a particular output file""" for msg in msglist: - f.write(msg + "\n") + LOG.info(msg) + # f.write(msg + "\n") else: def report(msglist): """Log messages by printing to stdout""" for msg in msglist: - print(msg) + LOG.info(msg) + # print(msg) return report @@ -392,9 +400,10 @@ def report(msglist): __version__ = 20150406 -def main(argv): +def main(): """ Main routine for the command-line interface to audfprint """ # Other globals set from command line + argv = sys.argv args = docopt.docopt(USAGE, version=__version__, argv=argv[1:]) # Figure which command was chosen @@ -409,7 +418,7 @@ def main(argv): cmd = cmdlist[0] # Setup output function - report = setup_reporter(args) # FFS + report = setup_reporter(args) # Keep track of wall time initticks = time_clock() @@ -467,7 +476,7 @@ def main(argv): matcher = setup_matcher(args) if cmd == 'match' else None filename_iter = filename_list_iterator( - args[''], args['--wavdir'], args['--wavext'], args['--list']) + args[''], args['--wavdir'], args['--wavext'], args['--list']) ####################### # Run the main commmand diff --git a/audfprint/audfprint_analyze.py b/audfprint/audfprint_analyze.py index 6dc253b..47e6d1a 100644 --- a/audfprint/audfprint_analyze.py +++ b/audfprint/audfprint_analyze.py @@ -21,11 +21,12 @@ import glob import time -import audio_read -# For utility, glob2hashtable -import hash_table from audfprint import LOG -import stft +from audfprint import audio_read +# For utility, glob2hashtable +from audfprint import hash_table + +from audfprint import stft # ############### Globals ############### # # Special extension indicating precomputed fingerprint diff --git a/audfprint/audfprint_match.py b/audfprint/audfprint_match.py index 29c0d64..61b10eb 100644 --- a/audfprint/audfprint_match.py +++ b/audfprint/audfprint_match.py @@ -23,9 +23,9 @@ from audfprint import LOG -import audfprint_analyze -import audio_read -import stft +from audfprint import audfprint_analyze +from audfprint import audio_read +from audfprint import stft def process_info(): diff --git a/tests/profile_audfprint.py b/tests/profile_audfprint.py deleted file mode 100644 index 8871f54..0000000 --- a/tests/profile_audfprint.py +++ /dev/null @@ -1,28 +0,0 @@ -# coding=utf-8 -import cProfile -import pstats - -# noinspection PyUnresolvedReferences -import audfprint - -argv = ["audfprint", "new", "-d", "tmp.fpdb", "--density", "200", - "data/Nine_Lives/01-Nine_Lives.mp3", - "data/Nine_Lives/02-Falling_In_Love.mp3", - "data/Nine_Lives/03-Hole_In_My_Soul.mp3", - "data/Nine_Lives/04-Taste_Of_India.mp3", - "data/Nine_Lives/05-Full_Circle.mp3", - "data/Nine_Lives/06-Something_s_Gotta_Give.mp3", - "data/Nine_Lives/07-Ain_t_That_A_Bitch.mp3", - "data/Nine_Lives/08-The_Farm.mp3", - "data/Nine_Lives/09-Crash.mp3", - "data/Nine_Lives/10-Kiss_Your_Past_Good-bye.mp3", - "data/Nine_Lives/11-Pink.mp3", - "data/Nine_Lives/12-Attitude_Adjustment.mp3", - "data/Nine_Lives/13-Fallen_Angels.mp3"] - -cProfile.run('audfprint.main(argv)', 'fpstats') - -p = pstats.Stats('fpstats') - -p.sort_stats('time') -p.print_stats(10) diff --git a/tests/test_profile_audfprint.py b/tests/test_profile_audfprint.py new file mode 100644 index 0000000..3f1c00f --- /dev/null +++ b/tests/test_profile_audfprint.py @@ -0,0 +1,31 @@ +# coding=utf-8 +import cProfile +import pstats + +# noinspection PyUnresolvedReferences +from audprint import audfprint + + + +def test_profile(): + argv = ["audfprint", "new", "-d", "tmp.fpdb", "--density", "200", + "data/Nine_Lives/01-Nine_Lives.mp3", + "data/Nine_Lives/02-Falling_In_Love.mp3", + "data/Nine_Lives/03-Hole_In_My_Soul.mp3", + "data/Nine_Lives/04-Taste_Of_India.mp3", + "data/Nine_Lives/05-Full_Circle.mp3", + "data/Nine_Lives/06-Something_s_Gotta_Give.mp3", + "data/Nine_Lives/07-Ain_t_That_A_Bitch.mp3", + "data/Nine_Lives/08-The_Farm.mp3", + "data/Nine_Lives/09-Crash.mp3", + "data/Nine_Lives/10-Kiss_Your_Past_Good-bye.mp3", + "data/Nine_Lives/11-Pink.mp3", + "data/Nine_Lives/12-Attitude_Adjustment.mp3", + "data/Nine_Lives/13-Fallen_Angels.mp3"] + + cProfile.run('audfprint.main(argv)', 'fpstats') + + p = pstats.Stats('fpstats') + + p.sort_stats('time') + p.print_stats(10) diff --git a/tests/profile_audfprint_match.py b/tests/test_profile_audfprint_match.py similarity index 100% rename from tests/profile_audfprint_match.py rename to tests/test_profile_audfprint_match.py diff --git a/tests/report_prof.py b/tests/test_report_prof.py similarity index 100% rename from tests/report_prof.py rename to tests/test_report_prof.py From 7f910bfe5ec2a766d3de349371732815ea05126a Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Wed, 10 Jul 2019 23:42:13 +0200 Subject: [PATCH 04/14] Create test_make.py --- tests/test_make.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/test_make.py diff --git a/tests/test_make.py b/tests/test_make.py new file mode 100644 index 0000000..ea29f40 --- /dev/null +++ b/tests/test_make.py @@ -0,0 +1,54 @@ +# just run the damn maketests +# https://github.com/dpwe/audfprint/blob/master/Makefile + + +def test_onecore(): + pass + + +def test_remove(): + pass + + +def test_fpdbase_pklz(): + pass + + +def test_test_onecore_precomp(): + pass + + +def test_test_onecore_newmerge(): + pass + + +def test_precompdir(): + pass + + +def test_onecore_precomppk(): + pass + + +def precomppkdir(): + pass + + +def test_mucore(): + pass + + +def test_fpdbase_mu_pklz(): + pass + + +def test_mucore_precomp(): + pass + + +def test_precompdir_mu(): + pass + + +def test_hash_mask(): + pass From 5db84b8c0e2a198c6a13367541fc35a38969c3b6 Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Thu, 11 Jul 2019 00:42:37 +0200 Subject: [PATCH 05/14] use relative imports. --- audfprint/audfprint_analyze.py | 8 ++++---- audfprint/audfprint_match.py | 8 ++++---- audfprint/audio_read.py | 12 ++++++------ audfprint/{audfprint.py => cli.py} | 0 audfprint/hash_table.py | 2 +- setup.py | 2 +- tests/conftest.py | 27 +++++++++++++++++++++++++++ tests/test_make.py | 3 +++ tests/test_profile_audfprint.py | 6 +++--- tests/test_profile_audfprint_match.py | 15 ++++++++++----- tests/test_report_prof.py | 13 +++++++------ 11 files changed, 66 insertions(+), 30 deletions(-) rename audfprint/{audfprint.py => cli.py} (100%) create mode 100644 tests/conftest.py diff --git a/audfprint/audfprint_analyze.py b/audfprint/audfprint_analyze.py index 47e6d1a..fc3c52c 100644 --- a/audfprint/audfprint_analyze.py +++ b/audfprint/audfprint_analyze.py @@ -21,12 +21,12 @@ import glob import time -from audfprint import LOG -from audfprint import audio_read +from . import LOG +from . import audio_read # For utility, glob2hashtable -from audfprint import hash_table +from . import hash_table -from audfprint import stft +from . import stft # ############### Globals ############### # # Special extension indicating precomputed fingerprint diff --git a/audfprint/audfprint_match.py b/audfprint/audfprint_match.py index 61b10eb..318f0e8 100644 --- a/audfprint/audfprint_match.py +++ b/audfprint/audfprint_match.py @@ -21,11 +21,11 @@ except: pass -from audfprint import LOG +from . import LOG -from audfprint import audfprint_analyze -from audfprint import audio_read -from audfprint import stft +from . import audfprint_analyze +from . import audio_read +from . import stft def process_info(): diff --git a/audfprint/audio_read.py b/audfprint/audio_read.py index 197c712..739904a 100644 --- a/audfprint/audio_read.py +++ b/audfprint/audio_read.py @@ -45,12 +45,12 @@ HAVE_FFMPEG = True def wavread(filename): - """Read in audio data from a wav file. Return d, sr.""" - # Read in wav file. - samplerate, wave_data = wav.read(filename) - # Normalize short ints to floats in range [-1..1). - data = np.asfarray(wave_data) / 32768.0 - return data, samplerate + """Read in audio data from a wav file. Return d, sr.""" + # Read in wav file. + samplerate, wave_data = wav.read(filename) + # Normalize short ints to floats in range [-1..1). + data = np.asfarray(wave_data) / 32768.0 + return data, samplerate def audio_read(filename, sr=None, channels=None): diff --git a/audfprint/audfprint.py b/audfprint/cli.py similarity index 100% rename from audfprint/audfprint.py rename to audfprint/cli.py diff --git a/audfprint/hash_table.py b/audfprint/hash_table.py index 28a59e6..b31f1e3 100644 --- a/audfprint/hash_table.py +++ b/audfprint/hash_table.py @@ -28,7 +28,7 @@ import cPickle as pickle # Py2 pickle_options = {} -from audfprint import LOG +from . import LOG # Current format version HT_VERSION = 20170724 diff --git a/setup.py b/setup.py index 945c489..dbac96b 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,7 @@ # pip to create the appropriate form of executable for the target platform. entry_points={ 'console_scripts': [ - 'audfprint=audfprint.audfprint:main', + 'audfprint=audfprint.cli:main', ] }, diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9b5cb4d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,27 @@ +import sys +import os + +import pytest + + +#fp = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'audfprint', 'audfprint') +#fp = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'audfprint') +#print(fp) +#print('ass') +# I dont like it.. +#sys.path.insert(1, fp) + + +@pytest.fixture(scope="module") +def test_dir(tmp_dir): + # A common test dir. + return str(tmp_dir) + + +def test_zomg(test_dir): + #args + path = os.path.join(test_dir, 'fpdbase.pklz') + args = ['--density', '100', '--skip-existing', + 'new', '--dbase', path, 'Nine_Lives/0*.mp3'] + + print(args) diff --git a/tests/test_make.py b/tests/test_make.py index ea29f40..134342e 100644 --- a/tests/test_make.py +++ b/tests/test_make.py @@ -1,6 +1,8 @@ # just run the damn maketests # https://github.com/dpwe/audfprint/blob/master/Makefile +# base python audfprint.py --density 100 --skip-existing + def test_onecore(): pass @@ -48,6 +50,7 @@ def test_mucore_precomp(): def test_precompdir_mu(): pass + # test_mucore_precomp def test_hash_mask(): diff --git a/tests/test_profile_audfprint.py b/tests/test_profile_audfprint.py index 3f1c00f..1680590 100644 --- a/tests/test_profile_audfprint.py +++ b/tests/test_profile_audfprint.py @@ -3,11 +3,11 @@ import pstats # noinspection PyUnresolvedReferences -from audprint import audfprint +from audfprint import cli -def test_profile(): +def _test_profile(): argv = ["audfprint", "new", "-d", "tmp.fpdb", "--density", "200", "data/Nine_Lives/01-Nine_Lives.mp3", "data/Nine_Lives/02-Falling_In_Love.mp3", @@ -23,7 +23,7 @@ def test_profile(): "data/Nine_Lives/12-Attitude_Adjustment.mp3", "data/Nine_Lives/13-Fallen_Angels.mp3"] - cProfile.run('audfprint.main(argv)', 'fpstats') + cProfile.run('cli.main(argv)', 'fpstats') p = pstats.Stats('fpstats') diff --git a/tests/test_profile_audfprint_match.py b/tests/test_profile_audfprint_match.py index fb859b0..f35b5ea 100644 --- a/tests/test_profile_audfprint_match.py +++ b/tests/test_profile_audfprint_match.py @@ -1,15 +1,20 @@ # coding=utf-8 +import sys + import cProfile import pstats # noinspection PyUnresolvedReferences +#from audfprint import cli import audfprint -argv = ["audfprint", "match", "-d", "fpdbase.pklz", "--density", "200", "query.mp3"] -cProfile.run('audfprint.main(argv)', 'fpmstats') +def zomg(): + argv = ["audfprint", "match", "-d", "fpdbase.pklz", "--density", "200", "query.mp3"] -p = pstats.Stats('fpmstats') + cProfile.run('audfprint.cli.main(argv)', 'fpmstats') + # dunno why this doest work. + p = pstats.Stats('fpmstats') -p.sort_stats('time') -p.print_stats(10) + p.sort_stats('time') + p.print_stats(10) diff --git a/tests/test_report_prof.py b/tests/test_report_prof.py index 815945d..b423274 100644 --- a/tests/test_report_prof.py +++ b/tests/test_report_prof.py @@ -5,9 +5,10 @@ import pstats import sys -prof_file = 'profile.out' -if len(sys.argv) > 1: - prof_file = sys.argv[1] -p = pstats.Stats(prof_file) -p.sort_stats('time') -p.print_stats(200) +def _zomg(): + prof_file = 'profile.out' + if len(sys.argv) > 1: + prof_file = sys.argv[1] + p = pstats.Stats(prof_file) + p.sort_stats('time') + p.print_stats(200) From 9be248dccb33376fa04a5667e817b77cd7c49d3b Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Thu, 11 Jul 2019 23:01:13 +0200 Subject: [PATCH 06/14] Run the tests on travis. --- .travis.yml | 18 ++++++++ tests/conftest.py | 59 ++++++++++++++++++++------- tests/test_make.py | 31 +++++++------- tests/test_profile_audfprint_match.py | 2 +- 4 files changed, 79 insertions(+), 31 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..206a6fb --- /dev/null +++ b/.travis.yml @@ -0,0 +1,18 @@ +language: python + +before_install: + - pip install --upgrade pip + - pip install --upgrade codecov pypandoc pytest pytest-cov pytest-mock +install: "pip install --no-cache-dir -e ." +# Lets do python tests later. +#script: py.test -vv -s --cov=$TRAVIS_BUILD_DIR/au/ --cov-report term-missing +script: make Makefile test +#after_success: codecov +#deploy: +# provider: pypi +# user: username +# password: +# secure: token +# on: +# tags: true +# branch: master \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 9b5cb4d..63bd86e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,27 +1,56 @@ import sys import os +import glob import pytest - -#fp = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'audfprint', 'audfprint') -#fp = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'audfprint') -#print(fp) -#print('ass') +fp = os.path.join(os.path.dirname(os.path.dirname(__file__))) # I dont like it.. -#sys.path.insert(1, fp) +sys.path.insert(1, fp) +TEST_ROOT = os.path.dirname(__file__) + +import audfprint -@pytest.fixture(scope="module") -def test_dir(tmp_dir): +@pytest.fixture(scope="session") +def testdir(tmpdir_factory): # A common test dir. - return str(tmp_dir) + p = tmpdir_factory.mktemp('zomg') + return str(p) + + +@pytest.fixture(scope="session") +def fpdbase_pklz_path(testdir): + return os.path.join(testdir, 'fpdbase.pklz') + + +@pytest.fixture(scope="session") +def fpdbase_pklz(fpdbase_pklz_path): + """Recreate fpdbase_pklz""" + fps = glob.glob('tests/data/Nine_Lives/0*.mp3') + args = ['audfprint', 'new', '--density', '100', '--skip-existing', + '--dbase', fpdbase_pklz_path] + fps + + sys.argv = args + audfprint.cli.main() + return fpdbase_pklz_path + + +@pytest.fixture(scope="session") +def default_args(): + return ["audfprint", "--density", "100", "--skip-existing"] + +@pytest.fixture(scope="session") +def query(): + return os.path.join(TEST_ROOT, 'data', 'query.mp3') -def test_zomg(test_dir): - #args - path = os.path.join(test_dir, 'fpdbase.pklz') - args = ['--density', '100', '--skip-existing', - 'new', '--dbase', path, 'Nine_Lives/0*.mp3'] - print(args) +def run(argv): + old_args = list(sys.argv) + try: + sys.argv = argv + audfprint.cli.main() + except Exception as e: + print(e) + sys.argv = old_args diff --git a/tests/test_make.py b/tests/test_make.py index 134342e..bac0e88 100644 --- a/tests/test_make.py +++ b/tests/test_make.py @@ -1,57 +1,58 @@ # just run the damn maketests # https://github.com/dpwe/audfprint/blob/master/Makefile -# base python audfprint.py --density 100 --skip-existing +from conftest import run -def test_onecore(): - pass +def test_onecore(fpdbase_pklz, query): + x = ["audfprint", "match", "--dbase", fpdbase_pklz, query] + run(x) -def test_remove(): +def _test_remove(): pass -def test_fpdbase_pklz(): +def _test_fpdbase_pklz(): pass -def test_test_onecore_precomp(): +def _test_test_onecore_precomp(): pass -def test_test_onecore_newmerge(): +def _test_test_onecore_newmerge(): pass -def test_precompdir(): +def _test_precompdir(): pass -def test_onecore_precomppk(): +def _test_onecore_precomppk(): pass -def precomppkdir(): +def _precomppkdir(): pass -def test_mucore(): +def _test_mucore(): pass -def test_fpdbase_mu_pklz(): +def _test_fpdbase_mu_pklz(): pass -def test_mucore_precomp(): +def _test_mucore_precomp(): pass -def test_precompdir_mu(): +def _test_precompdir_mu(): pass # test_mucore_precomp -def test_hash_mask(): +def _test_hash_mask(): pass diff --git a/tests/test_profile_audfprint_match.py b/tests/test_profile_audfprint_match.py index f35b5ea..15bffd1 100644 --- a/tests/test_profile_audfprint_match.py +++ b/tests/test_profile_audfprint_match.py @@ -9,7 +9,7 @@ import audfprint -def zomg(): +def _zomg(): argv = ["audfprint", "match", "-d", "fpdbase.pklz", "--density", "200", "query.mp3"] cProfile.run('audfprint.cli.main(argv)', 'fpmstats') From 26e5dc6064bcefcd6d553ebeb1bdb52a121f3524 Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Thu, 11 Jul 2019 23:40:03 +0200 Subject: [PATCH 07/14] import changes for the dpwe tools --- dpwe_builder.py | 6 +++--- dpwe_matcher.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dpwe_builder.py b/dpwe_builder.py index 877c0e3..5000483 100755 --- a/dpwe_builder.py +++ b/dpwe_builder.py @@ -25,7 +25,7 @@ import os import sys -import audfprint +from audfprint import cli try: # noinspection PyCompatibility @@ -72,7 +72,7 @@ "bucketsize:", bucketsize, "ncores:", ncores) # Ensure the database directory exists -audfprint.ensure_dir(dir4db) +cli.ensure_dir(dir4db) # Run the command argv = ["audfprint", "new", @@ -84,4 +84,4 @@ "--list", fileList4db] # Run audfprint -audfprint.main(argv) +cli.main(argv) diff --git a/dpwe_matcher.py b/dpwe_matcher.py index 437b3a9..849a54a 100755 --- a/dpwe_matcher.py +++ b/dpwe_matcher.py @@ -31,7 +31,7 @@ import os import sys -import audfprint +from audfprint.cli import main try: # noinspection PyCompatibility @@ -96,4 +96,4 @@ "--list", fileList4query] # Run audfprint -audfprint.main(argv) +main(argv) From 263e2d6da29131882ac19302b7a116770c8f3fb5 Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Thu, 11 Jul 2019 23:50:27 +0200 Subject: [PATCH 08/14] add missing pandoc and ffmpeg --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index 206a6fb..908febf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,12 @@ language: python +addons: + apt: + sources: + - sourceline: "ppa:mc3man/trusty-media" + update: true + packages: + - ffmpeg + - pandoc before_install: - pip install --upgrade pip From e4af25596ed80ee6aef7b759e98b032cc0cc695b Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Thu, 11 Jul 2019 23:57:00 +0200 Subject: [PATCH 09/14] Update Makefile Use the console entry point --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 06d3c06..1d00cec 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ # 2014-09-20 Dan Ellis dpwe@ee.columbia.edu #AUDFPRINT=python audfprint.py --skip-existing --continue-on-error -AUDFPRINT=python audfprint.py --density 100 --skip-existing +AUDFPRINT=audfprint --density 100 --skip-existing test: test_onecore test_onecore_precomp test_onecore_newmerge test_onecore_precomppk test_mucore test_mucore_precomp test_remove rm -rf precompdir precompdir_mu From 5d4c17c220d55311e4a199137f8013efecfbac01 Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Fri, 12 Jul 2019 00:06:36 +0200 Subject: [PATCH 10/14] Update Makefile damn make file, dunno wft im doing xD --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 1d00cec..a65086b 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ test_remove: fpdbase.pklz ${AUDFPRINT} list --dbase fpdbase.pklz ${AUDFPRINT} match --dbase fpdbase.pklz query.mp3 -fpdbase.pklz: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py +fpdbase.pklz: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py ${AUDFPRINT} new --dbase fpdbase.pklz Nine_Lives/0*.mp3 ${AUDFPRINT} add --dbase fpdbase.pklz Nine_Lives/1*.mp3 @@ -42,7 +42,7 @@ test_onecore_newmerge: precompdir ${AUDFPRINT} newmerge --dbase fpdbase2.pklz fpdbase0.pklz fpdbase1.pklz ${AUDFPRINT} match --dbase fpdbase2.pklz precompdir/query.afpt -precompdir: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py +precompdir: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py rm -rf precompdir mkdir precompdir ${AUDFPRINT} precompute --precompdir precompdir Nine_Lives/*.mp3 @@ -55,7 +55,7 @@ test_onecore_precomppk: precomppkdir ${AUDFPRINT} match --dbase fpdbase1.pklz precomppkdir/query.afpk rm -rf precomppkdir -precomppkdir: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py +precomppkdir: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py rm -rf precomppkdir mkdir precomppkdir ${AUDFPRINT} precompute --precompute-peaks --precompdir precomppkdir Nine_Lives/*.mp3 @@ -64,7 +64,7 @@ precomppkdir: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py test_mucore: fpdbase_mu.pklz ${AUDFPRINT} match --dbase fpdbase_mu.pklz --ncores 4 query.mp3 -fpdbase_mu.pklz: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py +fpdbase_mu.pklz: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py ${AUDFPRINT} new --dbase fpdbase_mu.pklz --ncores 4 Nine_Lives/0*.mp3 ${AUDFPRINT} add --dbase fpdbase_mu.pklz --ncores 4 Nine_Lives/1*.mp3 @@ -74,7 +74,7 @@ test_mucore_precomp: precompdir_mu ${AUDFPRINT} merge --dbase fpdbase_mu.pklz fpdbase_mu0.pklz ${AUDFPRINT} match --dbase fpdbase_mu.pklz --ncores 4 precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt -precompdir_mu: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py +precompdir_mu: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py rm -rf precompdir_mu mkdir precompdir_mu ${AUDFPRINT} precompute --ncores 4 --precompdir precompdir_mu Nine_Lives/*.mp3 From 2de95d95d548c50169c17f19f873ded9b7316597 Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Fri, 12 Jul 2019 00:12:08 +0200 Subject: [PATCH 11/14] Update Makefile dunno wtf im doing. but lets try this. --- Makefile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index a65086b..bb6fb2c 100644 --- a/Makefile +++ b/Makefile @@ -19,15 +19,15 @@ test_onecore: fpdbase.pklz ${AUDFPRINT} match --dbase fpdbase.pklz query.mp3 test_remove: fpdbase.pklz - ${AUDFPRINT} remove --dbase fpdbase.pklz Nine_Lives/05-Full_Circle.mp3 Nine_Lives/01-Nine_Lives.mp3 + ${AUDFPRINT} remove --dbase fpdbase.pklz tests/data/Nine_Lives/05-Full_Circle.mp3 tests/data/Nine_Lives/01-Nine_Lives.mp3 ${AUDFPRINT} list --dbase fpdbase.pklz - ${AUDFPRINT} add --dbase fpdbase.pklz Nine_Lives/01-Nine_Lives.mp3 Nine_Lives/05-Full_Circle.mp3 + ${AUDFPRINT} add --dbase fpdbase.pklz tests/data/Nine_Lives/01-Nine_Lives.mp3 tests/data/Nine_Lives/05-Full_Circle.mp3 ${AUDFPRINT} list --dbase fpdbase.pklz ${AUDFPRINT} match --dbase fpdbase.pklz query.mp3 fpdbase.pklz: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py - ${AUDFPRINT} new --dbase fpdbase.pklz Nine_Lives/0*.mp3 - ${AUDFPRINT} add --dbase fpdbase.pklz Nine_Lives/1*.mp3 + ${AUDFPRINT} new --dbase fpdbase.pklz tests/data/Nine_Lives/0*.mp3 + ${AUDFPRINT} add --dbase fpdbase.pklz tests/data/Nine_Lives/1*.mp3 test_onecore_precomp: precompdir ${AUDFPRINT} new --dbase fpdbase0.pklz precompdir/Nine_Lives/0* @@ -45,7 +45,7 @@ test_onecore_newmerge: precompdir precompdir: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py rm -rf precompdir mkdir precompdir - ${AUDFPRINT} precompute --precompdir precompdir Nine_Lives/*.mp3 + ${AUDFPRINT} precompute --precompdir precompdir tests/data/Nine_Lives/*.mp3 ${AUDFPRINT} precompute --precompdir precompdir --shifts 4 query.mp3 test_onecore_precomppk: precomppkdir @@ -58,15 +58,15 @@ test_onecore_precomppk: precomppkdir precomppkdir: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py rm -rf precomppkdir mkdir precomppkdir - ${AUDFPRINT} precompute --precompute-peaks --precompdir precomppkdir Nine_Lives/*.mp3 + ${AUDFPRINT} precompute --precompute-peaks --precompdir precomppkdir tests/data/Nine_Lives/*.mp3 ${AUDFPRINT} precompute --precompute-peaks --precompdir precomppkdir --shifts 4 query.mp3 test_mucore: fpdbase_mu.pklz ${AUDFPRINT} match --dbase fpdbase_mu.pklz --ncores 4 query.mp3 fpdbase_mu.pklz: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py - ${AUDFPRINT} new --dbase fpdbase_mu.pklz --ncores 4 Nine_Lives/0*.mp3 - ${AUDFPRINT} add --dbase fpdbase_mu.pklz --ncores 4 Nine_Lives/1*.mp3 + ${AUDFPRINT} new --dbase fpdbase_mu.pklz --ncores 4 tests/data/Nine_Lives/0*.mp3 + ${AUDFPRINT} add --dbase fpdbase_mu.pklz --ncores 4 tests/data/Nine_Lives/1*.mp3 test_mucore_precomp: precompdir_mu ${AUDFPRINT} new --dbase fpdbase_mu0.pklz --ncores 4 precompdir_mu/Nine_Lives/0* @@ -77,9 +77,9 @@ test_mucore_precomp: precompdir_mu precompdir_mu: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py rm -rf precompdir_mu mkdir precompdir_mu - ${AUDFPRINT} precompute --ncores 4 --precompdir precompdir_mu Nine_Lives/*.mp3 + ${AUDFPRINT} precompute --ncores 4 --precompdir precompdir_mu tests/data/Nine_Lives/*.mp3 ${AUDFPRINT} precompute --ncores 4 --precompdir precompdir_mu --shifts 4 query.mp3 query.mp3 query.mp3 query.mp3 query.mp3 query.mp3 test_hash_mask: - ${AUDFPRINT} new --dbase fpdbase.pklz --hashbits 16 Nine_Lives/*.mp3 + ${AUDFPRINT} new --dbase fpdbase.pklz --hashbits 16 tests/data/Nine_Lives/*.mp3 ${AUDFPRINT} match --dbase fpdbase.pklz query.mp3 From f59a73fb9cf78110fff1cdfe270bfba0bd93d8e5 Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Fri, 12 Jul 2019 00:41:57 +0200 Subject: [PATCH 12/14] Update Makefile fix query path. --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index bb6fb2c..cfa335e 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ precompdir: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.p rm -rf precompdir mkdir precompdir ${AUDFPRINT} precompute --precompdir precompdir tests/data/Nine_Lives/*.mp3 - ${AUDFPRINT} precompute --precompdir precompdir --shifts 4 query.mp3 + ${AUDFPRINT} precompute --precompdir precompdir --shifts 4 tests/data/query.mp3 test_onecore_precomppk: precomppkdir ${AUDFPRINT} new --dbase fpdbase0.pklz precomppkdir/Nine_Lives/0* @@ -59,10 +59,10 @@ precomppkdir: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match rm -rf precomppkdir mkdir precomppkdir ${AUDFPRINT} precompute --precompute-peaks --precompdir precomppkdir tests/data/Nine_Lives/*.mp3 - ${AUDFPRINT} precompute --precompute-peaks --precompdir precomppkdir --shifts 4 query.mp3 + ${AUDFPRINT} precompute --precompute-peaks --precompdir precomppkdir --shifts 4 tests/data/query.mp3 test_mucore: fpdbase_mu.pklz - ${AUDFPRINT} match --dbase fpdbase_mu.pklz --ncores 4 query.mp3 + ${AUDFPRINT} match --dbase fpdbase_mu.pklz --ncores 4 tests/data/query.mp3 fpdbase_mu.pklz: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py ${AUDFPRINT} new --dbase fpdbase_mu.pklz --ncores 4 tests/data/Nine_Lives/0*.mp3 @@ -78,8 +78,8 @@ precompdir_mu: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_matc rm -rf precompdir_mu mkdir precompdir_mu ${AUDFPRINT} precompute --ncores 4 --precompdir precompdir_mu tests/data/Nine_Lives/*.mp3 - ${AUDFPRINT} precompute --ncores 4 --precompdir precompdir_mu --shifts 4 query.mp3 query.mp3 query.mp3 query.mp3 query.mp3 query.mp3 + ${AUDFPRINT} precompute --ncores 4 --precompdir precompdir_mu --shifts 4 tests/data/query.mp3 tests/data/query.mp3 tests/data/query.mp3 tests/data/query.mp3 tests/data/query.mp3 tests/data/query.mp3 test_hash_mask: ${AUDFPRINT} new --dbase fpdbase.pklz --hashbits 16 tests/data/Nine_Lives/*.mp3 - ${AUDFPRINT} match --dbase fpdbase.pklz query.mp3 + ${AUDFPRINT} match --dbase fpdbase.pklz tests/data/query.mp3 From 58bf7c5bdc9b290c59b577c387f00f430679d2ae Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Sat, 13 Jul 2019 13:26:39 +0200 Subject: [PATCH 13/14] forgot some --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index cfa335e..8c9915a 100644 --- a/Makefile +++ b/Makefile @@ -16,14 +16,14 @@ test: test_onecore test_onecore_precomp test_onecore_newmerge test_onecore_preco rm -f fpdbase*.pklz test_onecore: fpdbase.pklz - ${AUDFPRINT} match --dbase fpdbase.pklz query.mp3 + ${AUDFPRINT} match --dbase fpdbase.pklz tests/data/query.mp3 test_remove: fpdbase.pklz ${AUDFPRINT} remove --dbase fpdbase.pklz tests/data/Nine_Lives/05-Full_Circle.mp3 tests/data/Nine_Lives/01-Nine_Lives.mp3 ${AUDFPRINT} list --dbase fpdbase.pklz ${AUDFPRINT} add --dbase fpdbase.pklz tests/data/Nine_Lives/01-Nine_Lives.mp3 tests/data/Nine_Lives/05-Full_Circle.mp3 ${AUDFPRINT} list --dbase fpdbase.pklz - ${AUDFPRINT} match --dbase fpdbase.pklz query.mp3 + ${AUDFPRINT} match --dbase fpdbase.pklz tests/data/query.mp3 fpdbase.pklz: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py ${AUDFPRINT} new --dbase fpdbase.pklz tests/data/Nine_Lives/0*.mp3 From 24c22587e5054fc12dd923eb35fa398753854849 Mon Sep 17 00:00:00 2001 From: Hellowlol Date: Sun, 14 Jul 2019 14:54:42 +0200 Subject: [PATCH 14/14] Some more changes to teh makefile. --- Makefile | 4 ++-- tests/conftest.py | 10 ++++++++-- tests/test_make.py | 9 ++++++--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 8c9915a..265f35d 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ test_onecore_newmerge: precompdir ${AUDFPRINT} newmerge --dbase fpdbase2.pklz fpdbase0.pklz fpdbase1.pklz ${AUDFPRINT} match --dbase fpdbase2.pklz precompdir/query.afpt -precompdir: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py +precompdir: audfprint/audfprint.cli.py audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py rm -rf precompdir mkdir precompdir ${AUDFPRINT} precompute --precompdir precompdir tests/data/Nine_Lives/*.mp3 @@ -74,7 +74,7 @@ test_mucore_precomp: precompdir_mu ${AUDFPRINT} merge --dbase fpdbase_mu.pklz fpdbase_mu0.pklz ${AUDFPRINT} match --dbase fpdbase_mu.pklz --ncores 4 precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt -precompdir_mu: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py +precompdir_mu: audfprint/cli.py audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py rm -rf precompdir_mu mkdir precompdir_mu ${AUDFPRINT} precompute --ncores 4 --precompdir precompdir_mu tests/data/Nine_Lives/*.mp3 diff --git a/tests/conftest.py b/tests/conftest.py index 63bd86e..3c9cc14 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,11 +28,17 @@ def fpdbase_pklz_path(testdir): def fpdbase_pklz(fpdbase_pklz_path): """Recreate fpdbase_pklz""" fps = glob.glob('tests/data/Nine_Lives/0*.mp3') + # new args = ['audfprint', 'new', '--density', '100', '--skip-existing', '--dbase', fpdbase_pklz_path] + fps - sys.argv = args - audfprint.cli.main() + run(args) + + # add + fps = glob.glob('tests/data/Nine_Lives/1*.mp3') + add_args = ['audfprint', 'add', '--dbase', fpdbase_pklz_path] + fps + + run(add_args) return fpdbase_pklz_path diff --git a/tests/test_make.py b/tests/test_make.py index bac0e88..57bc2aa 100644 --- a/tests/test_make.py +++ b/tests/test_make.py @@ -9,9 +9,12 @@ def test_onecore(fpdbase_pklz, query): run(x) -def _test_remove(): - pass - +def test_remove(fpdbase_pklz, query): + run(['audfprint', 'remove', '--dbase', fpdbase_pklz, 'tests/data/Nine_Lives/05-Full_Circle.mp3', 'tests/data/Nine_Lives/01-Nine_Lives.mp3']) + run(['audfprint', 'list', '--dbase', fpdbase_pklz]) + run(['audfprint', 'add', '--dbase', fpdbase_pklz, 'tests/data/Nine_Lives/01-Nine_Lives.mp3', 'tests/data/Nine_Lives/05-Full_Circle.mp3']) + run(['audfprint', 'list', '--dbase', fpdbase_pklz]) + run(['audfprint', 'match','--dbase', fpdbase_pklz, query]) def _test_fpdbase_pklz(): pass