diff --git a/.gitignore b/.gitignore index 5fd1337..08db47a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ var/ *.egg-info/ .installed.cfg *.egg +*.eggs # Installer logs pip-log.txt diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..908febf --- /dev/null +++ b/.travis.yml @@ -0,0 +1,26 @@ +language: python +addons: + apt: + sources: + - sourceline: "ppa:mc3man/trusty-media" + update: true + packages: + - ffmpeg + - pandoc + +before_install: + - pip install --upgrade pip + - pip install --upgrade codecov pypandoc pytest pytest-cov pytest-mock +install: "pip install --no-cache-dir -e ." +# Lets do python tests later. +#script: py.test -vv -s --cov=$TRAVIS_BUILD_DIR/au/ --cov-report term-missing +script: make Makefile test +#after_success: codecov +#deploy: +# provider: pypi +# user: username +# password: +# secure: token +# on: +# tags: true +# branch: master \ No newline at end of file diff --git a/Makefile b/Makefile index 06d3c06..265f35d 100644 --- a/Makefile +++ b/Makefile @@ -9,25 +9,25 @@ # 2014-09-20 Dan Ellis dpwe@ee.columbia.edu #AUDFPRINT=python audfprint.py --skip-existing --continue-on-error -AUDFPRINT=python audfprint.py --density 100 --skip-existing +AUDFPRINT=audfprint --density 100 --skip-existing test: test_onecore test_onecore_precomp test_onecore_newmerge test_onecore_precomppk test_mucore test_mucore_precomp test_remove rm -rf precompdir precompdir_mu rm -f fpdbase*.pklz test_onecore: fpdbase.pklz - ${AUDFPRINT} match --dbase fpdbase.pklz query.mp3 + ${AUDFPRINT} match --dbase fpdbase.pklz tests/data/query.mp3 test_remove: fpdbase.pklz - ${AUDFPRINT} remove --dbase fpdbase.pklz Nine_Lives/05-Full_Circle.mp3 Nine_Lives/01-Nine_Lives.mp3 + ${AUDFPRINT} remove --dbase fpdbase.pklz tests/data/Nine_Lives/05-Full_Circle.mp3 tests/data/Nine_Lives/01-Nine_Lives.mp3 ${AUDFPRINT} list --dbase fpdbase.pklz - ${AUDFPRINT} add --dbase fpdbase.pklz Nine_Lives/01-Nine_Lives.mp3 Nine_Lives/05-Full_Circle.mp3 + ${AUDFPRINT} add --dbase fpdbase.pklz tests/data/Nine_Lives/01-Nine_Lives.mp3 tests/data/Nine_Lives/05-Full_Circle.mp3 ${AUDFPRINT} list --dbase fpdbase.pklz - ${AUDFPRINT} match --dbase fpdbase.pklz query.mp3 + ${AUDFPRINT} match --dbase fpdbase.pklz tests/data/query.mp3 -fpdbase.pklz: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py - ${AUDFPRINT} new --dbase fpdbase.pklz Nine_Lives/0*.mp3 - ${AUDFPRINT} add --dbase fpdbase.pklz Nine_Lives/1*.mp3 +fpdbase.pklz: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py + ${AUDFPRINT} new --dbase fpdbase.pklz tests/data/Nine_Lives/0*.mp3 + ${AUDFPRINT} add --dbase fpdbase.pklz tests/data/Nine_Lives/1*.mp3 test_onecore_precomp: precompdir ${AUDFPRINT} new --dbase fpdbase0.pklz precompdir/Nine_Lives/0* @@ -42,11 +42,11 @@ test_onecore_newmerge: precompdir ${AUDFPRINT} newmerge --dbase fpdbase2.pklz fpdbase0.pklz fpdbase1.pklz ${AUDFPRINT} match --dbase fpdbase2.pklz precompdir/query.afpt -precompdir: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py +precompdir: audfprint/audfprint.cli.py audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py rm -rf precompdir mkdir precompdir - ${AUDFPRINT} precompute --precompdir precompdir Nine_Lives/*.mp3 - ${AUDFPRINT} precompute --precompdir precompdir --shifts 4 query.mp3 + ${AUDFPRINT} precompute --precompdir precompdir tests/data/Nine_Lives/*.mp3 + ${AUDFPRINT} precompute --precompdir precompdir --shifts 4 tests/data/query.mp3 test_onecore_precomppk: precomppkdir ${AUDFPRINT} new --dbase fpdbase0.pklz precomppkdir/Nine_Lives/0* @@ -55,18 +55,18 @@ test_onecore_precomppk: precomppkdir ${AUDFPRINT} match --dbase fpdbase1.pklz precomppkdir/query.afpk rm -rf precomppkdir -precomppkdir: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py +precomppkdir: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py rm -rf precomppkdir mkdir precomppkdir - ${AUDFPRINT} precompute --precompute-peaks --precompdir precomppkdir Nine_Lives/*.mp3 - ${AUDFPRINT} precompute --precompute-peaks --precompdir precomppkdir --shifts 4 query.mp3 + ${AUDFPRINT} precompute --precompute-peaks --precompdir precomppkdir tests/data/Nine_Lives/*.mp3 + ${AUDFPRINT} precompute --precompute-peaks --precompdir precomppkdir --shifts 4 tests/data/query.mp3 test_mucore: fpdbase_mu.pklz - ${AUDFPRINT} match --dbase fpdbase_mu.pklz --ncores 4 query.mp3 + ${AUDFPRINT} match --dbase fpdbase_mu.pklz --ncores 4 tests/data/query.mp3 -fpdbase_mu.pklz: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py - ${AUDFPRINT} new --dbase fpdbase_mu.pklz --ncores 4 Nine_Lives/0*.mp3 - ${AUDFPRINT} add --dbase fpdbase_mu.pklz --ncores 4 Nine_Lives/1*.mp3 +fpdbase_mu.pklz: audfprint audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py + ${AUDFPRINT} new --dbase fpdbase_mu.pklz --ncores 4 tests/data/Nine_Lives/0*.mp3 + ${AUDFPRINT} add --dbase fpdbase_mu.pklz --ncores 4 tests/data/Nine_Lives/1*.mp3 test_mucore_precomp: precompdir_mu ${AUDFPRINT} new --dbase fpdbase_mu0.pklz --ncores 4 precompdir_mu/Nine_Lives/0* @@ -74,12 +74,12 @@ test_mucore_precomp: precompdir_mu ${AUDFPRINT} merge --dbase fpdbase_mu.pklz fpdbase_mu0.pklz ${AUDFPRINT} match --dbase fpdbase_mu.pklz --ncores 4 precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt -precompdir_mu: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py +precompdir_mu: audfprint/cli.py audfprint/audfprint_analyze.py audfprint/audfprint_match.py audfprint/hash_table.py rm -rf precompdir_mu mkdir precompdir_mu - ${AUDFPRINT} precompute --ncores 4 --precompdir precompdir_mu Nine_Lives/*.mp3 - ${AUDFPRINT} precompute --ncores 4 --precompdir precompdir_mu --shifts 4 query.mp3 query.mp3 query.mp3 query.mp3 query.mp3 query.mp3 + ${AUDFPRINT} precompute --ncores 4 --precompdir precompdir_mu tests/data/Nine_Lives/*.mp3 + ${AUDFPRINT} precompute --ncores 4 --precompdir precompdir_mu --shifts 4 tests/data/query.mp3 tests/data/query.mp3 tests/data/query.mp3 tests/data/query.mp3 tests/data/query.mp3 tests/data/query.mp3 test_hash_mask: - ${AUDFPRINT} new --dbase fpdbase.pklz --hashbits 16 Nine_Lives/*.mp3 - ${AUDFPRINT} match --dbase fpdbase.pklz query.mp3 + ${AUDFPRINT} new --dbase fpdbase.pklz --hashbits 16 tests/data/Nine_Lives/*.mp3 + ${AUDFPRINT} match --dbase fpdbase.pklz tests/data/query.mp3 diff --git a/__init__.py b/__init__.py deleted file mode 100644 index 9bad579..0000000 --- a/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# coding=utf-8 diff --git a/audfprint/__init__.py b/audfprint/__init__.py new file mode 100644 index 0000000..66fd128 --- /dev/null +++ b/audfprint/__init__.py @@ -0,0 +1,5 @@ +# coding=utf-8 +import logging + +LOG = logging.getLogger(__name__) +LOG.addHandler(logging.NullHandler()) diff --git a/audfprint_analyze.py b/audfprint/audfprint_analyze.py similarity index 95% rename from audfprint_analyze.py rename to audfprint/audfprint_analyze.py index 3a73ec3..fc3c52c 100644 --- a/audfprint_analyze.py +++ b/audfprint/audfprint_analyze.py @@ -21,10 +21,12 @@ import glob import time -import audio_read +from . import LOG +from . import audio_read # For utility, glob2hashtable -import hash_table -import stft +from . import hash_table + +from . import stft # ############### Globals ############### # # Special extension indicating precomputed fingerprint @@ -86,7 +88,7 @@ def landmarks2hashes(landmarks): landmarks = np.array(landmarks) # Deal with special case of empty landmarks. if landmarks.shape[0] == 0: - return np.zeros((0, 2), dtype=np.int32) + return np.zeros((0, 2), dtype=np.int32) hashes = np.zeros((landmarks.shape[0], 2), dtype=np.int32) hashes[:, 0] = landmarks[:, 0] hashes[:, 1] = (((landmarks[:, 1] & B1_MASK) << B1_SHIFT) @@ -287,7 +289,8 @@ def find_peaks(self, d, sr): else: # The sgram is identically zero, i.e., the input signal was identically # zero. Not good, but let's let it through for now. - print("find_peaks: Warning: input signal is identically zero.") + LOG.debug('find_peaks: Warning: input signal is identically zero.') + # print("find_peaks: Warning: input signal is identically zero.") # High-pass filter onset emphasis # [:-1,] discards top bin (nyquist) of sgram so bins fit in 8 bits sgram = np.array([scipy.signal.lfilter([1, -1], @@ -329,7 +332,7 @@ def peaks2landmarks(self, pklist): for peak in peaks_at[col]: pairsthispeak = 0 for col2 in range(col + self.mindt, - min(scols, col + self.targetdt)): + min(scols, col + self.targetdt)): if pairsthispeak < self.maxpairsperpeak: for peak2 in peaks_at[col2]: if abs(peak2 - peak) < self.targetdf: @@ -357,11 +360,13 @@ def wavfile2peaks(self, filename, shifts=None): # [d, sr] = librosa.load(filename, sr=self.target_sr) d, sr = audio_read.audio_read(filename, sr=self.target_sr, channels=1) except Exception as e: # audioread.NoBackendError: - message = "wavfile2peaks: Error reading " + filename + message = "wavfile2peaks: Error reading %s" % filename if self.fail_on_error: - print(e) + # print(e) + LOG.error('%s', message) raise IOError(message) - print(message, "skipping") + LOG.debug('%s skipping', message) + # print(message, "skipping") d = [] sr = self.target_sr # Store duration in a global because it's hard to handle @@ -569,13 +574,16 @@ def glob2hashtable(pattern, density=20.0): totdur = 0.0 tothashes = 0 for ix, file_ in enumerate(filelist): - print(time.ctime(), "ingesting #", ix, ":", file_, "...") + LOG.debug('Ingesting #%s: %s ...', ix, file_) + # print(time.ctime(), "ingesting #", ix, ":", file_, "...") dur, nhash = g2h_analyzer.ingest(ht, file_) totdur += dur tothashes += nhash elapsedtime = time.clock() - initticks - print("Added", tothashes, "(", tothashes / totdur, "hashes/sec) at ", - elapsedtime / totdur, "x RT") + LOG.debug('Added %s %s hashes/sec at %s x RT', + tothashes, (tothashes / totdur), (elapsedtime / totdur)) + # print("Added", tothashes, "(", tothashes / totdur, "hashes/sec) at ", + # elapsedtime / totdur, "x RT") return ht diff --git a/audfprint_match.py b/audfprint/audfprint_match.py similarity index 96% rename from audfprint_match.py rename to audfprint/audfprint_match.py index d97026e..318f0e8 100644 --- a/audfprint_match.py +++ b/audfprint/audfprint_match.py @@ -21,9 +21,11 @@ except: pass -import audfprint_analyze -import audio_read -import stft +from . import LOG + +from . import audfprint_analyze +from . import audio_read +from . import stft def process_info(): @@ -40,7 +42,8 @@ def process_info(): def log(message): """ log info with stats """ - print('%s physmem=%s utime=%s %s' % (time.ctime(), process_info())) + LOG.debug('physmem=%s utime=%s %s' % process_info(), message) + # print('%s physmem=%s utime=%s %s' % (time.ctime(), process_info())) def encpowerof2(val): @@ -368,9 +371,11 @@ def match_file(self, analyzer, ht, filename, number=None): numberstring = "#%d" % number else: numberstring = "" - print(time.ctime(), "Analyzed", numberstring, filename, "of", - ('%.3f' % durd), "s " - "to", len(q_hashes), "hashes") + LOG.debug('Analyzed %s %s of %.3f s to %s hashes', + numberstring, filename, durd, len(q_hashes)) + # print(time.ctime(), "Analyzed", numberstring, filename, "of", + # ('%.3f' % durd), "s " + # "to", len(q_hashes), "hashes") # Run query rslts = self.match_hashes(ht, q_hashes) # Post filtering @@ -378,6 +383,7 @@ def match_file(self, analyzer, ht, filename, number=None): rslts = rslts[(-rslts[:, 2]).argsort(), :] return rslts[:self.max_returns, :], durd, len(q_hashes) + # WTF should we do about this. def file_match_to_msgs(self, analyzer, ht, qry, number=None): """ Perform a match on a single input file, return list of message strings """ @@ -459,10 +465,11 @@ def illustrate_match(self, analyzer, ht, filename): np.array([[x[1], x[2]] for x in mlms]).T, '.-r') # Add title - plt.title(filename + " : Matched as " + ht.names[results[0][0]] - + (" with %d of %d hashes" % (len(matchhashes), - len(q_hashes)))) + title = '%s :Matched as %s (with %d of %d hashes' % ( + ht.names[results[0][0]], len(matchhashes), len(q_hashes)) + # Display + plt.title(title) plt.show() # Return return results diff --git a/audio_read.py b/audfprint/audio_read.py similarity index 97% rename from audio_read.py rename to audfprint/audio_read.py index 197c712..739904a 100644 --- a/audio_read.py +++ b/audfprint/audio_read.py @@ -45,12 +45,12 @@ HAVE_FFMPEG = True def wavread(filename): - """Read in audio data from a wav file. Return d, sr.""" - # Read in wav file. - samplerate, wave_data = wav.read(filename) - # Normalize short ints to floats in range [-1..1). - data = np.asfarray(wave_data) / 32768.0 - return data, samplerate + """Read in audio data from a wav file. Return d, sr.""" + # Read in wav file. + samplerate, wave_data = wav.read(filename) + # Normalize short ints to floats in range [-1..1). + data = np.asfarray(wave_data) / 32768.0 + return data, samplerate def audio_read(filename, sr=None, channels=None): diff --git a/audfprint.py b/audfprint/cli.py old mode 100755 new mode 100644 similarity index 91% rename from audfprint.py rename to audfprint/cli.py index f5bb51b..f4735f1 --- a/audfprint.py +++ b/audfprint/cli.py @@ -9,6 +9,7 @@ 2014-05-25 Dan Ellis dpwe@ee.columbia.edu """ from __future__ import division, print_function +from logging.handlers import RotatingFileHandler # For reporting progress time import time @@ -21,12 +22,14 @@ import multiprocessing import joblib +from audfprint import LOG + # The actual analyzer class/code -import audfprint_analyze +from audfprint import audfprint_analyze # Access to match functions, used in command line interface -import audfprint_match +from audfprint import audfprint_match # My hash_table implementation -import hash_table +from audfprint import hash_table if sys.version_info[0] >= 3: @@ -119,7 +122,8 @@ def file_precompute_peaks_or_hashes(analyzer, filename, precompdir, def file_precompute(analyzer, filename, precompdir, type='peaks', skip_existing=False, strip_prefix=None): """ Perform precompute action for one file, return list of message strings """ - print(time.ctime(), "precomputing", type, "for", filename, "...") + LOG.info('%s precomputing %s for %s ...', time.ctime(), type, filename) + #print(time.ctime(), "precomputing", type, "for", filename, "...") hashes_not_peaks = (type == 'hashes') return file_precompute_peaks_or_hashes(analyzer, filename, precompdir, hashes_not_peaks=hashes_not_peaks, @@ -320,19 +324,26 @@ def setup_matcher(args): # Command to construct the reporter object def setup_reporter(args): """ Creates a logging function, either to stderr or file""" + # not really sure how we should handle this. + # but lets do this for now. + opfile = args['--opfile'] if opfile and len(opfile): - f = open(opfile, "w") + # Add a file handle to the msges is written to the log. + rfh = RotatingFileHandler(opfile, 'a', 512000, 3) + LOG.addHandler(rfh) def report(msglist): """Log messages to a particular output file""" for msg in msglist: - f.write(msg + "\n") + LOG.info(msg) + # f.write(msg + "\n") else: def report(msglist): """Log messages by printing to stdout""" for msg in msglist: - print(msg) + LOG.info(msg) + # print(msg) return report @@ -389,9 +400,10 @@ def report(msglist): __version__ = 20150406 -def main(argv): +def main(): """ Main routine for the command-line interface to audfprint """ # Other globals set from command line + argv = sys.argv args = docopt.docopt(USAGE, version=__version__, argv=argv[1:]) # Figure which command was chosen @@ -435,10 +447,9 @@ def main(argv): # Check that the output directory can be created before we start ensure_dir(os.path.split(dbasename)[0]) # Create a new hash table - hash_tab = hash_table.HashTable( - hashbits=int(args['--hashbits']), - depth=int(args['--bucketsize']), - maxtime=(1 << int(args['--maxtimebits']))) + hash_tab = hash_table.HashTable(hashbits=int(args['--hashbits']), + depth=int(args['--bucketsize']), + maxtime=(1 << int(args['--maxtimebits']))) # Set its samplerate param if analyzer: hash_tab.params['samplerate'] = analyzer.target_sr @@ -446,12 +457,14 @@ def main(argv): else: # Load existing hash table file (add, match, merge) if args['--verbose']: - report([time.ctime() + " Reading hash table " + dbasename]) + LOG.debug('Reading hashtable %s', dbasename) + # report([time.ctime() + " Reading hash table " + dbasename]) hash_tab = hash_table.HashTable(dbasename) if analyzer and 'samplerate' in hash_tab.params \ and hash_tab.params['samplerate'] != analyzer.target_sr: # analyzer.target_sr = hash_tab.params['samplerate'] - print("db samplerate overridden to ", analyzer.target_sr) + LOG.info('db samplerate overridden to %s', analyzer.target_sr) + # print("db samplerate overridden to ", analyzer.target_sr) else: # The command IS precompute # dummy empty hash table @@ -463,7 +476,7 @@ def main(argv): matcher = setup_matcher(args) if cmd == 'match' else None filename_iter = filename_list_iterator( - args[''], args['--wavdir'], args['--wavext'], args['--list']) + args[''], args['--wavdir'], args['--wavext'], args['--list']) ####################### # Run the main commmand @@ -488,10 +501,13 @@ def main(argv): elapsedtime = time_clock() - initticks if analyzer and analyzer.soundfiletotaldur > 0.: - print("Processed " - + "%d files (%.1f s total dur) in %.1f s sec = %.3f x RT" \ - % (analyzer.soundfilecount, analyzer.soundfiletotaldur, - elapsedtime, (elapsedtime / analyzer.soundfiletotaldur))) + LOG.debug('Processed %d files %.1f s total dur in %.1f s sec = %.3f x RT', + analyzer.soundfilecount, analyzer.soundfiletotaldur, + elapsedtime, (elapsedtime / analyzer.soundfiletotaldur)) + # print("Processed " + # + "%d files (%.1f s total dur) in %.1f s sec = %.3f x RT" \ + # % (analyzer.soundfilecount, analyzer.soundfiletotaldur, + # elapsedtime, (elapsedtime / analyzer.soundfiletotaldur))) # Save the hash table file if it has been modified if hash_tab and hash_tab.dirty: diff --git a/hash_table.py b/audfprint/hash_table.py similarity index 93% rename from hash_table.py rename to audfprint/hash_table.py index 1f853c7..b31f1e3 100644 --- a/hash_table.py +++ b/audfprint/hash_table.py @@ -28,6 +28,7 @@ import cPickle as pickle # Py2 pickle_options = {} +from . import LOG # Current format version HT_VERSION = 20170724 @@ -192,9 +193,11 @@ def save(self, name, params=None, file_object=None): nhashes = sum(self.counts) # Report the proportion of dropped hashes (overfull table) dropped = nhashes - sum(np.minimum(self.depth, self.counts)) - print("Saved fprints for", sum(n is not None for n in self.names), - "files (", nhashes, "hashes) to", name, - "(%.2f%% dropped)" % (100.0 * dropped / max(1, nhashes))) + LOG.debug('Saved fprint for %s files (%s hashes) to %s (%.2f%% dropped)', + sum(n is not None for n in self.names), nhashes, name, (100.0 * dropped / max(1, nhashes))) + # print("Saved fprints for", sum(n is not None for n in self.names), + # "files (", nhashes, "hashes) to", name, + # "(%.2f%% dropped)" % (100.0 * dropped / max(1, nhashes))) def load(self, name): """ Read either pklz or mat-format hash table file """ @@ -206,9 +209,11 @@ def load(self, name): nhashes = sum(self.counts) # Report the proportion of dropped hashes (overfull table) dropped = nhashes - sum(np.minimum(self.depth, self.counts)) - print("Read fprints for", sum(n is not None for n in self.names), - "files (", nhashes, "hashes) from", name, - "(%.2f%% dropped)" % (100.0 * dropped / max(1, nhashes))) + LOG.debug('Read fprint for %s files (%s hashes) from %s (%.2f%% dropped)', + sum(n is not None for n in self.names), nhashes, name, (100.0 * dropped / max(1, nhashes))) + # print("Read fprints for", sum(n is not None for n in self.names), + # "files (", nhashes, "hashes) from", name, + # "(%.2f%% dropped)" % (100.0 * dropped / max(1, nhashes))) def load_pkl(self, name, file_object=None): """ Read hash table values from pickle file . """ @@ -361,7 +366,8 @@ def remove(self, name): self.names[id_] = None self.hashesperid[id_] = 0 self.dirty = True - print("Removed", name, "(", hashes_removed, "hashes).") + LOG.debug('Removed %s (%s) hashes.', name, hashes_removed) + # print("Removed", name, "(", hashes_removed, "hashes).") def retrieve(self, name): """Return an np.array of (time, hash) pairs found in the table.""" @@ -384,8 +390,9 @@ def retrieve(self, name): def list(self, print_fn=None): """ List all the known items. """ - if not print_fn: - print_fn = print + # print_fn is not in use but keeping it for compat. + for name, count in zip(self.names, self.hashesperid): if name: - print_fn(name + " (" + str(count) + " hashes)") + LOG.info('%s (%s) hashes', name, count) + #print_fn(name + " (" + str(count) + " hashes)") diff --git a/audfprint/stft.py b/audfprint/stft.py new file mode 100644 index 0000000..bc29fc9 --- /dev/null +++ b/audfprint/stft.py @@ -0,0 +1,93 @@ +"""Provide stft to avoid librosa dependency. + +This implementation is based on routines from +https://github.com/tensorflow/models/blob/master/research/audioset/mel_features.py +""" + +from __future__ import division + +import numpy as np + + +def frame(data, window_length, hop_length): + """Convert array into a sequence of successive possibly overlapping frames. + + An n-dimensional array of shape (num_samples, ...) is converted into an + (n+1)-D array of shape (num_frames, window_length, ...), where each frame + starts hop_length points after the preceding one. + + This is accomplished using stride_tricks, so the original data is not + copied. However, there is no zero-padding, so any incomplete frames at the + end are not included. + + Args: + data: np.array of dimension N >= 1. + window_length: Number of samples in each frame. + hop_length: Advance (in samples) between each window. + + Returns: + (N+1)-D np.array with as many rows as there are complete frames that can be + extracted. + """ + num_samples = data.shape[0] + num_frames = 1 + ((num_samples - window_length) // hop_length) + shape = (num_frames, window_length) + data.shape[1:] + strides = (data.strides[0] * hop_length,) + data.strides + return np.lib.stride_tricks.as_strided(data, shape=shape, strides=strides) + + +def periodic_hann(window_length): + """Calculate a "periodic" Hann window. + + The classic Hann window is defined as a raised cosine that starts and + ends on zero, and where every value appears twice, except the middle + point for an odd-length window. Matlab calls this a "symmetric" window + and np.hanning() returns it. However, for Fourier analysis, this + actually represents just over one cycle of a period N-1 cosine, and + thus is not compactly expressed on a length-N Fourier basis. Instead, + it's better to use a raised cosine that ends just before the final + zero value - i.e. a complete cycle of a period-N cosine. Matlab + calls this a "periodic" window. This routine calculates it. + + Args: + window_length: The number of points in the returned window. + + Returns: + A 1D np.array containing the periodic hann window. + """ + return 0.5 - (0.5 * np.cos(2 * np.pi / window_length * np.arange(window_length))) + + +def stft(signal, n_fft, hop_length=None, window=None): + """Calculate the short-time Fourier transform. + + Args: + signal: 1D np.array of the input time-domain signal. + n_fft: Size of the FFT to apply. + hop_length: Advance (in samples) between each frame passed to FFT. Defaults + to half the window length. + window: Length of each block of samples to pass to FFT, or vector of window + values. Defaults to n_fft. + + Returns: + 2D np.array where each column contains the complex values of the + fft_length/2+1 unique values of the FFT for the corresponding frame of + input samples ("spectrogram transposition"). + """ + if window is None: + window = n_fft + if isinstance(window, (int, float)): + # window holds the window length, need to make the actual window. + window = periodic_hann(int(window)) + window_length = len(window) + if not hop_length: + hop_length = window_length // 2 + # Default librosa STFT behavior. + pad_mode = "reflect" + signal = np.pad(signal, (n_fft // 2), mode=pad_mode) + frames = frame(signal, window_length, hop_length) + # Apply frame window to each frame. We use a periodic Hann (cosine of period + # window_length) instead of the symmetric Hann of np.hanning (period + # window_length-1). + windowed_frames = frames * window + return np.fft.rfft(windowed_frames, n_fft).transpose() diff --git a/dpwe_builder.py b/dpwe_builder.py index 877c0e3..5000483 100755 --- a/dpwe_builder.py +++ b/dpwe_builder.py @@ -25,7 +25,7 @@ import os import sys -import audfprint +from audfprint import cli try: # noinspection PyCompatibility @@ -72,7 +72,7 @@ "bucketsize:", bucketsize, "ncores:", ncores) # Ensure the database directory exists -audfprint.ensure_dir(dir4db) +cli.ensure_dir(dir4db) # Run the command argv = ["audfprint", "new", @@ -84,4 +84,4 @@ "--list", fileList4db] # Run audfprint -audfprint.main(argv) +cli.main(argv) diff --git a/dpwe_matcher.py b/dpwe_matcher.py index 437b3a9..849a54a 100755 --- a/dpwe_matcher.py +++ b/dpwe_matcher.py @@ -31,7 +31,7 @@ import os import sys -import audfprint +from audfprint.cli import main try: # noinspection PyCompatibility @@ -96,4 +96,4 @@ "--list", fileList4query] # Run audfprint -audfprint.main(argv) +main(argv) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..dbac96b --- /dev/null +++ b/setup.py @@ -0,0 +1,95 @@ +# coding=UTF-8 + +""" +https://packaging.python.org/en/latest/distributing.html +https://github.com/pypa/sampleproject +""" + +from setuptools import setup, find_packages + +# Possibly convert the README.md to .rst-format +try: + import pypandoc + README = pypandoc.convert('README.md', 'rst') +except ImportError: + print("warning: pypandoc module not found, could not convert Markdown to RST") + README = open('README.md', 'r').read() + + +f = open('requirements.txt', 'r').readlines() +REQ = [] +for line in f: + if line and not line.startswith('#'): + REQ.append(line) + + +setup( + name='audfprint', + + # Version number is automatically extracted from Git + # https://pypi.python.org/pypi/setuptools_scm + # https://packaging.python.org/en/latest/single_source_version.html + use_scm_version=True, + setup_requires=['setuptools_scm', 'pypandoc'], + # version='0.0.1', + + description='Landmark-based audio fingerprinting.', + long_description=README, + + # The project's main homepage. + url='https://github.com/dpwe/audfprint', + + # Author details + author='Dan Ellis', + author_email='some@email.com', + + # Choose your license + license='GPL3', + + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + 'Development Status :: 4 - Beta', + + # Indicate who your project is intended for + 'Intended Audience :: End Users/Desktop', + 'Environment :: Console', + 'Topic :: Multimedia :: Audio', + + # Pick your license as you wish (should match "license" above) + #'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + + # Specify the Python versions you support here. + 'Programming Language :: Python :: 2.7, 3.6', + ], + + # What does your project relate to? + keywords='landmark audioprint, fingerprint', + + # You can just specify the packages manually here if your project is + # simple. Or you can use find_packages(). + packages=find_packages(exclude=['tests']), + include_package_data=True, + + + # List run-time dependencies here. These will be installed by pip when + # your project is installed. For an analysis of "install_requires" vs pip's + # requirements files see: + # https://packaging.python.org/en/latest/requirements.html + install_requires=REQ, + + # List additional groups of dependencies here (e.g. development + # dependencies). You can install these using the following syntax, + # for example: + # $ pip install -e .[dev,test] + + # To provide executable scripts, use entry points in preference to the + # "scripts" keyword. Entry points provide cross-platform support and allow + # pip to create the appropriate form of executable for the target platform. + entry_points={ + 'console_scripts': [ + 'audfprint=audfprint.cli:main', + ] + }, + + +) diff --git a/stft.py b/stft.py deleted file mode 100644 index 6dcd772..0000000 --- a/stft.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Provide stft to avoid librosa dependency. - -This implementation is based on routines from -https://github.com/tensorflow/models/blob/master/research/audioset/mel_features.py -""" - -from __future__ import division - -import numpy as np - - -def frame(data, window_length, hop_length): - """Convert array into a sequence of successive possibly overlapping frames. - - An n-dimensional array of shape (num_samples, ...) is converted into an - (n+1)-D array of shape (num_frames, window_length, ...), where each frame - starts hop_length points after the preceding one. - - This is accomplished using stride_tricks, so the original data is not - copied. However, there is no zero-padding, so any incomplete frames at the - end are not included. - - Args: - data: np.array of dimension N >= 1. - window_length: Number of samples in each frame. - hop_length: Advance (in samples) between each window. - - Returns: - (N+1)-D np.array with as many rows as there are complete frames that can be - extracted. - """ - num_samples = data.shape[0] - num_frames = 1 + ((num_samples - window_length) // hop_length) - shape = (num_frames, window_length) + data.shape[1:] - strides = (data.strides[0] * hop_length,) + data.strides - return np.lib.stride_tricks.as_strided(data, shape=shape, strides=strides) - - -def periodic_hann(window_length): - """Calculate a "periodic" Hann window. - - The classic Hann window is defined as a raised cosine that starts and - ends on zero, and where every value appears twice, except the middle - point for an odd-length window. Matlab calls this a "symmetric" window - and np.hanning() returns it. However, for Fourier analysis, this - actually represents just over one cycle of a period N-1 cosine, and - thus is not compactly expressed on a length-N Fourier basis. Instead, - it's better to use a raised cosine that ends just before the final - zero value - i.e. a complete cycle of a period-N cosine. Matlab - calls this a "periodic" window. This routine calculates it. - - Args: - window_length: The number of points in the returned window. - - Returns: - A 1D np.array containing the periodic hann window. - """ - return 0.5 - (0.5 * np.cos(2 * np.pi / window_length * - np.arange(window_length))) - - -def stft(signal, n_fft, hop_length=None, window=None): - """Calculate the short-time Fourier transform. - - Args: - signal: 1D np.array of the input time-domain signal. - n_fft: Size of the FFT to apply. - hop_length: Advance (in samples) between each frame passed to FFT. Defaults - to half the window length. - window: Length of each block of samples to pass to FFT, or vector of window - values. Defaults to n_fft. - - Returns: - 2D np.array where each column contains the complex values of the - fft_length/2+1 unique values of the FFT for the corresponding frame of - input samples ("spectrogram transposition"). - """ - if window is None: - window = n_fft - if isinstance(window, (int, float)): - # window holds the window length, need to make the actual window. - window = periodic_hann(int(window)) - window_length = len(window) - if not hop_length: - hop_length = window_length // 2 - # Default librosa STFT behavior. - pad_mode = 'reflect' - signal = np.pad(signal, (n_fft // 2), mode=pad_mode) - frames = frame(signal, window_length, hop_length) - # Apply frame window to each frame. We use a periodic Hann (cosine of period - # window_length) instead of the symmetric Hann of np.hanning (period - # window_length-1). - windowed_frames = frames * window - return np.fft.rfft(windowed_frames, n_fft).transpose() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..3c9cc14 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,62 @@ +import sys +import os +import glob + +import pytest + +fp = os.path.join(os.path.dirname(os.path.dirname(__file__))) +# I dont like it.. +sys.path.insert(1, fp) +TEST_ROOT = os.path.dirname(__file__) + +import audfprint + + +@pytest.fixture(scope="session") +def testdir(tmpdir_factory): + # A common test dir. + p = tmpdir_factory.mktemp('zomg') + return str(p) + + +@pytest.fixture(scope="session") +def fpdbase_pklz_path(testdir): + return os.path.join(testdir, 'fpdbase.pklz') + + +@pytest.fixture(scope="session") +def fpdbase_pklz(fpdbase_pklz_path): + """Recreate fpdbase_pklz""" + fps = glob.glob('tests/data/Nine_Lives/0*.mp3') + # new + args = ['audfprint', 'new', '--density', '100', '--skip-existing', + '--dbase', fpdbase_pklz_path] + fps + + run(args) + + # add + fps = glob.glob('tests/data/Nine_Lives/1*.mp3') + add_args = ['audfprint', 'add', '--dbase', fpdbase_pklz_path] + fps + + run(add_args) + return fpdbase_pklz_path + + +@pytest.fixture(scope="session") +def default_args(): + return ["audfprint", "--density", "100", "--skip-existing"] + + +@pytest.fixture(scope="session") +def query(): + return os.path.join(TEST_ROOT, 'data', 'query.mp3') + + +def run(argv): + old_args = list(sys.argv) + try: + sys.argv = argv + audfprint.cli.main() + except Exception as e: + print(e) + sys.argv = old_args diff --git a/tests/profile_audfprint.py b/tests/profile_audfprint.py deleted file mode 100644 index 8871f54..0000000 --- a/tests/profile_audfprint.py +++ /dev/null @@ -1,28 +0,0 @@ -# coding=utf-8 -import cProfile -import pstats - -# noinspection PyUnresolvedReferences -import audfprint - -argv = ["audfprint", "new", "-d", "tmp.fpdb", "--density", "200", - "data/Nine_Lives/01-Nine_Lives.mp3", - "data/Nine_Lives/02-Falling_In_Love.mp3", - "data/Nine_Lives/03-Hole_In_My_Soul.mp3", - "data/Nine_Lives/04-Taste_Of_India.mp3", - "data/Nine_Lives/05-Full_Circle.mp3", - "data/Nine_Lives/06-Something_s_Gotta_Give.mp3", - "data/Nine_Lives/07-Ain_t_That_A_Bitch.mp3", - "data/Nine_Lives/08-The_Farm.mp3", - "data/Nine_Lives/09-Crash.mp3", - "data/Nine_Lives/10-Kiss_Your_Past_Good-bye.mp3", - "data/Nine_Lives/11-Pink.mp3", - "data/Nine_Lives/12-Attitude_Adjustment.mp3", - "data/Nine_Lives/13-Fallen_Angels.mp3"] - -cProfile.run('audfprint.main(argv)', 'fpstats') - -p = pstats.Stats('fpstats') - -p.sort_stats('time') -p.print_stats(10) diff --git a/tests/profile_audfprint_match.py b/tests/profile_audfprint_match.py deleted file mode 100644 index fb859b0..0000000 --- a/tests/profile_audfprint_match.py +++ /dev/null @@ -1,15 +0,0 @@ -# coding=utf-8 -import cProfile -import pstats - -# noinspection PyUnresolvedReferences -import audfprint - -argv = ["audfprint", "match", "-d", "fpdbase.pklz", "--density", "200", "query.mp3"] - -cProfile.run('audfprint.main(argv)', 'fpmstats') - -p = pstats.Stats('fpmstats') - -p.sort_stats('time') -p.print_stats(10) diff --git a/tests/test_make.py b/tests/test_make.py new file mode 100644 index 0000000..57bc2aa --- /dev/null +++ b/tests/test_make.py @@ -0,0 +1,61 @@ +# just run the damn maketests +# https://github.com/dpwe/audfprint/blob/master/Makefile + +from conftest import run + + +def test_onecore(fpdbase_pklz, query): + x = ["audfprint", "match", "--dbase", fpdbase_pklz, query] + run(x) + + +def test_remove(fpdbase_pklz, query): + run(['audfprint', 'remove', '--dbase', fpdbase_pklz, 'tests/data/Nine_Lives/05-Full_Circle.mp3', 'tests/data/Nine_Lives/01-Nine_Lives.mp3']) + run(['audfprint', 'list', '--dbase', fpdbase_pklz]) + run(['audfprint', 'add', '--dbase', fpdbase_pklz, 'tests/data/Nine_Lives/01-Nine_Lives.mp3', 'tests/data/Nine_Lives/05-Full_Circle.mp3']) + run(['audfprint', 'list', '--dbase', fpdbase_pklz]) + run(['audfprint', 'match','--dbase', fpdbase_pklz, query]) + +def _test_fpdbase_pklz(): + pass + + +def _test_test_onecore_precomp(): + pass + + +def _test_test_onecore_newmerge(): + pass + + +def _test_precompdir(): + pass + + +def _test_onecore_precomppk(): + pass + + +def _precomppkdir(): + pass + + +def _test_mucore(): + pass + + +def _test_fpdbase_mu_pklz(): + pass + + +def _test_mucore_precomp(): + pass + + +def _test_precompdir_mu(): + pass + # test_mucore_precomp + + +def _test_hash_mask(): + pass diff --git a/tests/test_profile_audfprint.py b/tests/test_profile_audfprint.py new file mode 100644 index 0000000..1680590 --- /dev/null +++ b/tests/test_profile_audfprint.py @@ -0,0 +1,31 @@ +# coding=utf-8 +import cProfile +import pstats + +# noinspection PyUnresolvedReferences +from audfprint import cli + + + +def _test_profile(): + argv = ["audfprint", "new", "-d", "tmp.fpdb", "--density", "200", + "data/Nine_Lives/01-Nine_Lives.mp3", + "data/Nine_Lives/02-Falling_In_Love.mp3", + "data/Nine_Lives/03-Hole_In_My_Soul.mp3", + "data/Nine_Lives/04-Taste_Of_India.mp3", + "data/Nine_Lives/05-Full_Circle.mp3", + "data/Nine_Lives/06-Something_s_Gotta_Give.mp3", + "data/Nine_Lives/07-Ain_t_That_A_Bitch.mp3", + "data/Nine_Lives/08-The_Farm.mp3", + "data/Nine_Lives/09-Crash.mp3", + "data/Nine_Lives/10-Kiss_Your_Past_Good-bye.mp3", + "data/Nine_Lives/11-Pink.mp3", + "data/Nine_Lives/12-Attitude_Adjustment.mp3", + "data/Nine_Lives/13-Fallen_Angels.mp3"] + + cProfile.run('cli.main(argv)', 'fpstats') + + p = pstats.Stats('fpstats') + + p.sort_stats('time') + p.print_stats(10) diff --git a/tests/test_profile_audfprint_match.py b/tests/test_profile_audfprint_match.py new file mode 100644 index 0000000..15bffd1 --- /dev/null +++ b/tests/test_profile_audfprint_match.py @@ -0,0 +1,20 @@ +# coding=utf-8 +import sys + +import cProfile +import pstats + +# noinspection PyUnresolvedReferences +#from audfprint import cli +import audfprint + + +def _zomg(): + argv = ["audfprint", "match", "-d", "fpdbase.pklz", "--density", "200", "query.mp3"] + + cProfile.run('audfprint.cli.main(argv)', 'fpmstats') + # dunno why this doest work. + p = pstats.Stats('fpmstats') + + p.sort_stats('time') + p.print_stats(10) diff --git a/tests/report_prof.py b/tests/test_report_prof.py similarity index 64% rename from tests/report_prof.py rename to tests/test_report_prof.py index 815945d..b423274 100644 --- a/tests/report_prof.py +++ b/tests/test_report_prof.py @@ -5,9 +5,10 @@ import pstats import sys -prof_file = 'profile.out' -if len(sys.argv) > 1: - prof_file = sys.argv[1] -p = pstats.Stats(prof_file) -p.sort_stats('time') -p.print_stats(200) +def _zomg(): + prof_file = 'profile.out' + if len(sys.argv) > 1: + prof_file = sys.argv[1] + p = pstats.Stats(prof_file) + p.sort_stats('time') + p.print_stats(200)