Shift to Python 3 and add a viewer

.gitattributes

@@ -0,0 +1,2 @@
+*.p text eol=lf
+*.npy text eol=lf

bat_eval/cnn_helpers.py

@@ -11,10 +11,11 @@ def aligned_malloc(shape, dtype, alignment=16):
     alignment is required memory alignment in bytes
     """
     itemsize = np.dtype(dtype).itemsize
-    extra = alignment / itemsize
+    extra = alignment // itemsize
     size = np.prod(shape)
+    #print(size, type(size), extra, type(extra))
     buf = np.empty(size + extra, dtype=dtype)
-    ofs = (-buf.ctypes.data % alignment) / itemsize
+    ofs = (-buf.ctypes.data % alignment) // itemsize
     aa = buf[ofs:ofs+size].reshape(shape)
     assert (aa.ctypes.data % alignment) == 0
     assert (aa.flags['C_CONTIGUOUS']) == True
@@ -120,7 +121,7 @@ def fully_connected_as_corr(ip, filters, bias):
     """
 
     # create DxHxsliding_width views of input - similar to corr2d
-    sliding_width = filters.shape[0] / np.prod(ip.shape[:2])
+    sliding_width = filters.shape[0] // np.prod(ip.shape[:2])
     op = view_as_windows(ip, (ip.shape[0],ip.shape[1],sliding_width))
     op = op.reshape((np.prod(op.shape[:3]), np.prod(op.shape[3:])))
 

bat_eval/cpu_detection.py

@@ -1,13 +1,12 @@
+import pickle
+import warnings
 import numpy as np
 from scipy.ndimage import zoom
 from scipy.ndimage.filters import gaussian_filter1d
-import cPickle as pickle
-import time
 
 from spectrogram import Spectrogram
 import cnn_helpers as ch
 
-import warnings
 warnings.simplefilter("ignore", UserWarning)
 
 try:
@@ -32,8 +31,8 @@ def __init__(self, weight_file, params_file):
         params_file is the path to the network parameters
         """
 
-        self.weights = np.load(weight_file)
-        if not all([weight.dtype==np.float32 for weight in self.weights]):
+        self.weights = np.load(weight_file, encoding='bytes', allow_pickle=True)
+        if not all([weight.dtype == np.float32 for weight in self.weights]):
             for i in range(self.weights.shape[0]):
                 self.weights[i] = self.weights[i].astype(np.float32)
 
@@ -99,10 +98,15 @@ def create_spec(self, audio, sampling_rate):
                                     max_freq=self.max_freq, min_freq=self.min_freq)
         hspec = self.sp.process_spectrogram(hspec, denoise_spec=self.denoise,
                                     smooth_spec=self.smooth_spec)
-        nsize = (np.ceil(hspec.shape[0]/2.0).astype(int), np.ceil(hspec.shape[1]/2.0).astype(int))
-        spec = ch.aligned_malloc(nsize, np.float32)
-
-        zoom(hspec, 0.5, output=spec, order=1)
+        #nsize = (np.ceil(hspec.shape[0]/2.0).astype(int), 
+        #                 np.ceil(hspec.shape[1]/2.0).astype(int))
+        nsize = zoom(hspec, 0.5, order=1).shape #dm edit
+        spec = ch.aligned_malloc(nsize, np.float32) 
+        try:
+            zoom(hspec, 0.5, output=spec, order=1)
+        except Exception as e:
+            print(e, hspec.shape, nsize)
+            #print(spec, hspec, nsize,e)
         return spec
 
 
@@ -138,7 +142,7 @@ def eval_network_1_dense(self, ip):
         prob = prob - np.amax(prob, axis=1, keepdims=True)
         prob = np.exp(prob)
         prob = prob[:, 1] / prob.sum(1)
-        prob = np.hstack((prob, np.zeros((ip.shape[1]/4)-prob.shape[0], dtype=np.float32)))
+        prob = np.hstack((prob, np.zeros((ip.shape[1]//4)-prob.shape[0], dtype=np.float32)))
 
         return prob
 
@@ -169,7 +173,6 @@ def eval_network_2_dense(self, ip):
         prob = prob - np.amax(prob, axis=1, keepdims=True)
         prob = np.exp(prob)
         prob = prob[:, 1] / prob.sum(1)
-        prob = np.hstack((prob, np.zeros((ip.shape[1]/4)-prob.shape[0], dtype=np.float32)))
+        prob = np.hstack((prob, np.zeros((ip.shape[1]//4)-prob.shape[0], dtype=np.float32)))
 
         return prob
-

bat_eval/myskimage.py

@@ -6,7 +6,6 @@
 were also copied to as dependencies of "gaussian" function.
 """
 from __future__ import division
-import numbers
 import collections as coll
 import numpy as np
 from scipy import ndimage as ndi
@@ -41,7 +40,7 @@ def warn(msg):
 
 
 def img_as_float(image):
-    dtype=np.float32
+    dtype = np.float32
     force_copy = False
 
     """

bat_eval/nms_slow.py

@@ -59,8 +59,8 @@ def test_nms():
     pos, prob = nms_1d(y, win_size, y.shape[0])
     pos_f, prob_f = nms_fast.nms_1d(y, win_size, y.shape[0])
 
-    print 'diff between implementations =', 1-np.isclose(prob_f, prob).mean()
-    print 'diff between implementations =', 1-np.isclose(pos_f, pos).mean()
+    print( 'diff between implementations =', 1-np.isclose(prob_f, prob).mean())
+    print( 'diff between implementations =', 1-np.isclose(pos_f, pos).mean())
 
     plt.close('all')
     plt.plot(y)

bat_eval/run_detector.py

@@ -14,15 +14,15 @@ def read_audio(file_name, do_time_expansion, chunk_size, win_size):
     try:
         samp_rate_orig, audio = mywavfile.read(file_name)
     except:
-        print '  Error reading file'
+        print( '  Error reading file')
         return True, None, None, None, None
 
     # convert to mono if stereo
     if len(audio.shape) == 2:
-        print '  Warning: stereo file. Just taking left channel.'
+        print('  Warning: stereo file. Just taking left channel.')
         audio = audio[:, 0]
     file_dur = audio.shape[0] / float(samp_rate_orig)
-    print '  dur', round(file_dur,3), '(secs) , fs', samp_rate_orig
+    print('  dur', round(file_dur,3), '(secs) , fs', samp_rate_orig)
 
     # original model is trained on time expanded data
     samp_rate = samp_rate_orig
@@ -75,15 +75,45 @@ def run_model(det, audio, file_dur, samp_rate, detection_thresh, max_num_calls=0
 
 if __name__ == "__main__":
 
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Process some bat files.')
+    parser.add_argument('--threshold', type=float, default=0.9,
+                   help='detection threshold')
+    parser.add_argument('--datadir',  
+                   help='root directory for output')
+    parser.add_argument('--resultsdir',  default='results',
+                   help='directory in which to place the results')
+    parser.add_argument('--resfile',  default='op_file.csv',
+                   help='results file name')
+    parser.add_argument('--timeexp',  default=True,
+                   help='Apply time expansion')
+    parser.add_argument('--saveres',  default=True,
+                   help='Save results files')
+
+    args = parser.parse_args()
+    print(args)
     # params
-    detection_thresh = 0.95  # make this smaller if you want more calls
-    do_time_expansion = True # if audio is already time expanded set this to False
-    save_res = True
+    detection_thresh = 0.9
+    if 'threshold' in args:
+	    detection_thresh =args.threshold  # make this smaller if you want more calls
+    do_time_expansion = args.timeexp # if audio is already time expanded set this to False
+    save_res = args.saveres
 
     # load data
-    data_dir = 'wavs/'       # this is the path to your audio files
-    op_ann_dir = 'results/'  # this where your results will be saved
-    op_file_name_total = op_ann_dir + 'op_file.csv'
+    data_dir = 'C:\\Users/David/Documents/Bat recordings/Audiomoth/moth4/Stonebarrow/'      # this is the path to your audio files
+    if 'datadir' in args:
+        data_dir=args.datadir  
+    outdir = 'results3/'
+    if 'resultsdir' in args:
+        outdir = args.resultsdir
+    op_ann_dir = os.path.join(data_dir , outdir)  # this where your results will be saved
+
+    os.makedirs(op_ann_dir, exist_ok=True)
+    outfile = 'op_file.csv'
+    if 'resfile' in args:
+	    outfile = args.resfile
+    op_file_name_total = os.path.join(op_ann_dir, outfile)
     if not os.path.isdir(op_ann_dir):
         os.makedirs(op_ann_dir)
 
@@ -100,7 +130,7 @@ def run_model(det, audio, file_dur, samp_rate, detection_thresh, max_num_calls=0
     for file_cnt, file_name in enumerate(audio_files):
 
         file_name_root = file_name[len(data_dir):]
-        print '\n', file_cnt+1, 'of', len(audio_files), '\t', file_name_root
+        print( '\n', file_cnt+1, 'of', len(audio_files), '\t', file_name_root)
 
         # read audio file - skip file if can't read it
         read_fail, audio, file_dur, samp_rate, samp_rate_orig = read_audio(file_name,
@@ -114,9 +144,9 @@ def run_model(det, audio, file_dur, samp_rate, detection_thresh, max_num_calls=0
                                         detection_thresh)
         toc = time.time()
 
-        print '  detection time', round(toc-tic, 3), '(secs)'
+        print( '  detection time', round(toc-tic, 3), '(secs)')
         num_calls = len(det_time)
-        print '  ' + str(num_calls) + ' calls found'
+        print( '  ' + str(num_calls) + ' calls found')
 
         # save results
         if save_res:
@@ -133,8 +163,8 @@ def run_model(det, audio, file_dur, samp_rate, detection_thresh, max_num_calls=0
 
     # save results for all files to large csv
     if save_res and (len(results) > 0):
-        print '\nsaving results to', op_file_name_total
+        print( '\nsaving results to', op_file_name_total)
         wo.save_to_txt(op_file_name_total, results)
     else:
-        print 'no detections to save'
+        print( 'no detections to save')
 

bat_train/classifier.py

@@ -18,7 +18,7 @@ def __init__(self, params_):
         elif self.params.classification_model == 'segment':
             self.model = seg.SegmentAudio(self.params)
         else:
-            print 'Invalid model specified'
+            print('Invalid model specified')
 
     def save_features(self, files):
         self.model.save_features(files)
@@ -34,9 +34,9 @@ def train(self, files, gt_pos, durations):
 
         # hard negative mining
         if self.params.num_hard_negative_mining > 0 and self.params.classification_model != 'segment':
-            print '\nhard negative mining'
+            print( '\nhard negative mining')
             for hn in range(self.params.num_hard_negative_mining):
-                print '\thmn round', hn
+                print( '\thmn round', hn)
                 positions, class_labels = self.do_hnm(files, gt_pos, durations, positions, class_labels)
                 self.model.train(positions, class_labels, files, durations)
 

bat_train/cls_audio_forest.py

@@ -37,7 +37,7 @@ def train(self, positions, class_labels, files, durations):
         # flatten list of lists and set to correct output
         features = np.vstack(feats)
         labels = np.vstack(labs)
-        print 'train size', features.shape
+        print( 'train size', features.shape)
         self.forest.train(features, labels, False)
 
     def test(self, file_name=None, file_duration=None, audio_samples=None, sampling_rate=None):
@@ -67,7 +67,7 @@ def create_or_load_features(self, file_name=None, audio_samples=None, sampling_r
             features = compute_features(audio_samples, sampling_rate, self.params)
         else:
             if self.params.load_features_from_file:
-                features = np.load(self.params.feature_dir + file_name + '.npy')
+                features = np.load(self.params.feature_dir + file_name + '.npy', encoding='bytes')
             else:
                 sampling_rate, audio_samples = wavfile.read(self.params.audio_dir + file_name + '.wav')
                 features = compute_features(audio_samples, sampling_rate, self.params)

bat_train/cls_cnn.py

@@ -38,7 +38,7 @@ def train(self, positions, class_labels, files, durations):
         # flatten list of lists and set to correct output size
         features = np.vstack(feats)
         labels = np.vstack(labs).astype(np.uint8)[:,0]
-        print 'train size', features.shape
+        print( 'train size', features.shape)
 
         # train network
         input_var = theano.tensor.tensor4('inputs')
@@ -90,7 +90,7 @@ def create_or_load_features(self, file_name=None, audio_samples=None, sampling_r
             features = compute_features(audio_samples, sampling_rate, self.params)
         else:
             if self.params.load_features_from_file:
-                features = np.load(self.params.feature_dir + file_name + '.npy')
+                features = np.load(self.params.feature_dir + file_name + '.npy', encoding='bytes')
             else:
                 sampling_rate, audio_samples = wavfile.read(self.params.audio_dir + file_name + '.wav')
                 features = compute_features(audio_samples, sampling_rate, self.params)
@@ -120,7 +120,7 @@ def build_cnn(ip_size, input_var, net_type):
     elif net_type == 'small':
         net = network_sm(ip_size, input_var)
     else:
-        print 'Error: network not defined'
+        print( 'Error: network not defined')
     return net
 
 def network_big(ip_size, input_var):

bat_train/create_results.py

@@ -11,7 +11,7 @@
 def plot_prec_recall(alg_name, recall, precision, nms_prob=None):
     # average precision
     ave_prec = evl.calc_average_precision(recall, precision)
-    print 'average precision (area) = %.3f ' % ave_prec
+    print( 'average precision (area) = %.3f ' % ave_prec)
 
     # recall at 95% precision
     desired_precision = 0.95
@@ -20,7 +20,7 @@ def plot_prec_recall(alg_name, recall, precision, nms_prob=None):
     else:
         recall_at_precision = 0
 
-    print 'recall at', int(desired_precision*100), '% precision = ', "%.3f" % recall_at_precision
+    print( 'recall at', int(desired_precision*100), '% precision = ', "%.3f" % recall_at_precision)
     plt.plot([0, 1.02], [desired_precision, desired_precision], 'b:', linewidth=1)
     plt.plot([recall_at_precision, recall_at_precision], [0, desired_precision], 'b:', linewidth=1)
 

bat_train/evaluate.py

@@ -12,7 +12,7 @@ def compute_error_auc(op_str, gt, pred, prob):
     fpr, tpr, thresholds = roc_curve(gt, pred)
     roc_auc = auc(fpr, tpr)
 
-    print op_str, ', class acc = %.3f, ROC AUC = %.3f' % (class_acc, roc_auc)
+    print( op_str, ', class acc = %.3f, ROC AUC = %.3f' % (class_acc, roc_auc))
     #return class_acc, roc_auc
 
 

bat_train/export_detector_weights.py

@@ -3,7 +3,7 @@
 can use it.
 """
 
-import cPickle as pickle
+import pickle
 from lasagne.layers.helper import get_all_param_values, get_output_shape, set_all_param_values
 import numpy as np
 import lasagne
@@ -13,16 +13,16 @@
 
 save_detector = False
 
-print 'saving detector'
+print( 'saving detector')
 model_dir = 'results/models/'
 model_file = model_dir + 'test_set_norfolk.mod'
-print model_file
+print( model_file)
 
 mod = pickle.load(open(model_file))
 
 weights = get_all_param_values(mod.model.network['prob'])
 np.save(model_file[:-4], weights)
-print 'weights shape', len(weights)
+print( 'weights shape', len(weights))
 
 # save detection params
 mod_params = {'win_size':0, 'chunk_size':0, 'max_freq':0, 'min_freq':0,

bat_train/nms_slow.py

@@ -59,8 +59,8 @@ def test_nms():
     pos, prob = nms_1d(y, win_size, y.shape[0])
     pos_f, prob_f = nms_fast.nms_1d(y, win_size, y.shape[0])
 
-    print 'diff between implementations =', 1-np.isclose(prob_f, prob).mean()
-    print 'diff between implementations =', 1-np.isclose(pos_f, pos).mean()
+    print( 'diff between implementations =', 1-np.isclose(prob_f, prob).mean())
+    print( 'diff between implementations =', 1-np.isclose(pos_f, pos).mean())
 
     plt.close('all')
     plt.plot(y)

bat_train/random_forest.py

@@ -254,7 +254,7 @@ def optimize_node(self, x_local, y_local, node):
 
 ## Parallel training helper - used to train trees in parallel
 def train_forest_helper(t_id, X, Y, params, seed):
-    #print 'tree', t_id
+    #print( 'tree', t_id)
     np.random.seed(seed)
     tree = Tree(t_id, params)
     tree.train(X, Y)
@@ -277,9 +277,9 @@ def train(self, X, Y, delete_old_trees):
             self.trees.extend(Parallel(n_jobs=-1)(delayed(train_forest_helper)(t_id, X, Y, self.params, seeds[t_id])
                                              for t_id in range(self.params.num_trees)))
         else:
-            #print 'Standard training'
+            #print( 'Standard training')
             for t_id in range(self.params.num_trees):
-                print 'tree', t_id
+                print( 'tree', t_id)
                 tree = Tree(t_id, self.params)
                 tree.train(X, Y)
                 self.trees.append(tree)