diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 0bb54cec489..ea6a9a0fa67 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -107,6 +107,7 @@ else() add_subdirectory(server) add_subdirectory(quantize) add_subdirectory(vad-speech-segments) + add_subdirectory(stream-resumable) if (WHISPER_SDL2) add_subdirectory(stream) add_subdirectory(command) diff --git a/examples/stream-resumable/CMakeLists.txt b/examples/stream-resumable/CMakeLists.txt new file mode 100644 index 00000000000..b460dedf355 --- /dev/null +++ b/examples/stream-resumable/CMakeLists.txt @@ -0,0 +1,8 @@ +set(TARGET whisper-stream-resumable) +add_executable(${TARGET} stream-resumable.cpp) + +include(DefaultTargetOptions) + +target_link_libraries(${TARGET} PRIVATE common whisper ${FFMPEG_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) + +install(TARGETS ${TARGET} RUNTIME) diff --git a/examples/stream-resumable/README.md b/examples/stream-resumable/README.md new file mode 100644 index 00000000000..122688fe086 --- /dev/null +++ b/examples/stream-resumable/README.md @@ -0,0 +1,71 @@ +# whisper.cpp/examples/stream-resumable + +Reference for the **resumable / asynchronous streaming** API: feed audio +incrementally and transcribe it next to recording, at full quality, without the +sliding-window re-decoding (and output divergence) of `examples/stream`. + +Each 30s window is decoded exactly once; the seek position and rolling text +context are persisted in the `whisper_state` across calls, so already-emitted +segments are never revised. The output is consistent with a single batch run. + +## API + +```c +whisper_resumable_reset_with_state(ctx, state); + +// as audio arrives (16 kHz mono f32): +whisper_append_audio_with_state(ctx, state, pcm, n); +int n_new = whisper_full_resumable_with_state(ctx, state, params, /*finalize=*/false); +// ... consume the n_new newly produced segments ... + +// when the stream ends, flush the trailing (< 30s) window: +whisper_full_resumable_with_state(ctx, state, params, /*finalize=*/true); +``` + +`whisper_full_resumable_with_state` decodes every **complete** 30s window +currently available and returns the number of new segments. When less than 30s +of undecoded audio is available (and `finalize == false`) it returns 0 without +decoding a partial window ("need more audio"). + +## Threading + +The model weights (`whisper_context`) are shared read-only; each concurrent +stream uses its own `whisper_state` (allocated with `whisper_init_state`). This +example runs one **producer** thread (audio source) and one **worker** thread +(inference) connected by a PCM queue, so transcription is decoupled from +capture. In your application, replace the file-reading producer with your +microphone / network source. + +## Mel normalization + +`whisper_full_params.mel_norm_mode` selects how the log-mel is normalized: + +- `WHISPER_MEL_NORM_GLOBAL` (default): normalize against the maximum seen across + all audio appended so far. This matches `whisper_full()` / batch behavior + exactly only when the whole signal is appended before the first decode; when + decoding incrementally, early windows use the running max rather than the + whole-signal max (the difference is negligible for typical speech). +- `WHISPER_MEL_NORM_WINDOW`: normalize each window against a reference level + with an envelope follower — instantaneous attack (so loud passages never + over-drive) and an exponential release with a half-life in audio seconds + (`mel_norm_half_life`), so a brief silence does not amplify background noise + and a steady background source that stops is forgotten only gradually. Useful + for live audio with varying levels. + +## Usage + +```bash +# build (no SDL required) +cmake -B build -DWHISPER_BUILD_EXAMPLES=ON +cmake --build build --target whisper-stream-resumable -j + +# transcribe a 16 kHz mono WAV, simulating a live source in real time +./build/bin/whisper-stream-resumable \ + -m models/ggml-base.en.bin -f samples/jfk.wav --realtime + +# live mel normalization with a 2s release half-life +./build/bin/whisper-stream-resumable \ + -m models/ggml-base.en.bin -f samples/jfk.wav --window-norm --half-life 2.0 +``` + +Run with `--help` for all options. diff --git a/examples/stream-resumable/stream-resumable.cpp b/examples/stream-resumable/stream-resumable.cpp new file mode 100644 index 00000000000..428630dc0bf --- /dev/null +++ b/examples/stream-resumable/stream-resumable.cpp @@ -0,0 +1,229 @@ +// Resumable / asynchronous streaming transcription example. +// +// This demonstrates the resumable whisper API: +// - whisper_append_audio_with_state() feed PCM incrementally +// - whisper_full_resumable_with_state(.., false) decode complete 30s windows +// - whisper_full_resumable_with_state(.., true) flush the trailing window +// +// Unlike examples/stream (which repeatedly calls whisper_full() on a sliding +// window and therefore re-decodes overlapping audio, producing output that +// changes between iterations), this decodes each window exactly once, resumes +// the seek position and the rolling text context from the state, and never +// revises already-emitted segments. The result is consistent with a single +// batch run. +// +// The design that matters for a real application: +// - ONE producer thread captures audio and pushes PCM into a queue. +// - ONE worker thread owns a dedicated whisper_state and runs inference, +// decoupled from capture so transcription can run at full quality while +// recording continues. +// - The model weights (whisper_context) are shared read-only; each worker +// would use its own whisper_state. +// +// Here the "producer" reads a WAV file and (optionally) paces it in real time +// to simulate a live source. In your application, replace the producer with +// your microphone / network audio source and push 16 kHz mono f32 PCM. + +#include "common.h" +#include "common-whisper.h" +#include "whisper.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// ----------------------------------------------------------------------------- +// A minimal thread-safe PCM queue (single producer, single consumer). +// ----------------------------------------------------------------------------- +class pcm_queue { +public: + void push(const float * data, size_t n) { + { + std::lock_guard lock(mtx_); + buf_.insert(buf_.end(), data, data + n); + } + cv_.notify_one(); + } + + // Signal that no more audio will arrive. + void close() { + { + std::lock_guard lock(mtx_); + closed_ = true; + } + cv_.notify_all(); + } + + // Block until at least one sample is available or the queue is closed. + // Drains everything currently buffered into `out`. Returns false only when + // the queue is closed AND drained (i.e. the stream has ended). + bool pop_all(std::vector & out) { + std::unique_lock lock(mtx_); + cv_.wait(lock, [&] { return !buf_.empty() || closed_; }); + out.assign(buf_.begin(), buf_.end()); + buf_.clear(); + return !out.empty() || !closed_; + } + +private: + std::mutex mtx_; + std::condition_variable cv_; + std::deque buf_; + bool closed_ = false; +}; + +// ----------------------------------------------------------------------------- +struct cli_params { + std::string model = "models/ggml-base.en.bin"; + std::string fname; + std::string language = "en"; + int n_threads = std::min(4, (int) std::thread::hardware_concurrency()); + int chunk_ms = 1000; // how much audio the producer emits per push + bool realtime = false; // pace the producer to wall-clock time + bool window_norm = false; // use the per-window (live) mel normalization + float half_life = 3.0f; // release half-life in seconds (window norm) + bool translate = false; +}; + +static void print_usage(const char * argv0) { + fprintf(stderr, + "usage: %s -m -f [options]\n" + " -m, --model PATH model path (default: models/ggml-base.en.bin)\n" + " -f, --file PATH input WAV (16 kHz mono); required\n" + " -l, --language LANG spoken language or 'auto' (default: en)\n" + " -t, --threads N inference threads (default: %d)\n" + " --chunk-ms N producer chunk size in ms (default: 1000)\n" + " --realtime pace the producer to real time (simulate live)\n" + " --window-norm per-window mel normalization (live AGC) instead of global\n" + " --half-life S release half-life in seconds for --window-norm (default: 3.0)\n" + " --translate translate to English\n", + argv0, std::min(4, (int) std::thread::hardware_concurrency())); +} + +static bool parse_args(int argc, char ** argv, cli_params & p) { + for (int i = 1; i < argc; i++) { + std::string a = argv[i]; + auto next = [&](const char * name) -> std::string { + if (i + 1 >= argc) { fprintf(stderr, "missing value for %s\n", name); exit(1); } + return argv[++i]; + }; + if (a == "-m" || a == "--model") p.model = next("model"); + else if (a == "-f" || a == "--file") p.fname = next("file"); + else if (a == "-l" || a == "--language") p.language = next("language"); + else if (a == "-t" || a == "--threads") p.n_threads = std::stoi(next("threads")); + else if (a == "--chunk-ms") p.chunk_ms = std::stoi(next("chunk-ms")); + else if (a == "--realtime") p.realtime = true; + else if (a == "--window-norm") p.window_norm = true; + else if (a == "--half-life") p.half_life = std::stof(next("half-life")); + else if (a == "--translate") p.translate = true; + else if (a == "-h" || a == "--help") { print_usage(argv[0]); exit(0); } + else { fprintf(stderr, "unknown argument: %s\n", a.c_str()); return false; } + } + if (p.fname.empty()) { print_usage(argv[0]); return false; } + return true; +} + +// Print every segment that was produced since `already_printed`, return the new total. +static int print_new_segments(whisper_state * state, int already_printed) { + const int n = whisper_full_n_segments_from_state(state); + for (int i = already_printed; i < n; i++) { + const int64_t t0 = whisper_full_get_segment_t0_from_state(state, i); + const int64_t t1 = whisper_full_get_segment_t1_from_state(state, i); + const char * text = whisper_full_get_segment_text_from_state(state, i); + printf("[%s --> %s]%s\n", to_timestamp(t0).c_str(), to_timestamp(t1).c_str(), text); + } + fflush(stdout); + return n; +} + +int main(int argc, char ** argv) { + cli_params p; + if (!parse_args(argc, argv, p)) return 1; + + // load the audio up front (the producer thread streams it out below) + std::vector pcm; + std::vector> pcms; + if (!read_audio_data(p.fname, pcm, pcms, /*stereo=*/false)) { + fprintf(stderr, "error: failed to read audio '%s'\n", p.fname.c_str()); + return 1; + } + + // shared, read-only context (model weights) + whisper_context_params cparams = whisper_context_default_params(); + whisper_context * ctx = whisper_init_from_file_with_params_no_state(p.model.c_str(), cparams); + if (!ctx) { + fprintf(stderr, "error: failed to load model '%s'\n", p.model.c_str()); + return 1; + } + + // dedicated inference state for the worker (one per concurrent stream) + whisper_state * state = whisper_init_state(ctx); + if (!state) { + fprintf(stderr, "error: failed to init state\n"); + whisper_free(ctx); + return 1; + } + + whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY); + wparams.print_progress = false; + wparams.print_realtime = false; + wparams.print_timestamps = false; + wparams.translate = p.translate; + wparams.language = p.language.c_str(); + wparams.n_threads = p.n_threads; + wparams.no_context = false; // carry rolling context across windows + if (p.window_norm) { + wparams.mel_norm_mode = WHISPER_MEL_NORM_WINDOW; + wparams.mel_norm_half_life = p.half_life; + } else { + wparams.mel_norm_mode = WHISPER_MEL_NORM_GLOBAL; + } + + whisper_resumable_reset_with_state(ctx, state); + + pcm_queue queue; + std::atomic failed{false}; + + // ---- worker thread: append audio + decode complete windows as they arrive ---- + std::thread worker([&] { + int printed = 0; + std::vector chunk; + while (queue.pop_all(chunk)) { + if (chunk.empty()) continue; // woke up but nothing buffered yet + if (whisper_append_audio_with_state(ctx, state, chunk.data(), (int) chunk.size()) != 0) { + failed = true; return; + } + const int ret = whisper_full_resumable_with_state(ctx, state, wparams, /*finalize=*/false); + if (ret < 0) { failed = true; return; } + printed = print_new_segments(state, printed); + } + // stream ended: flush the trailing (< 30s) window + const int ret = whisper_full_resumable_with_state(ctx, state, wparams, /*finalize=*/true); + if (ret < 0) { failed = true; return; } + print_new_segments(state, printed); + }); + + // ---- producer: stream the file out in chunks (this is where a mic would feed in) ---- + const int chunk_n = (p.chunk_ms * WHISPER_SAMPLE_RATE) / 1000; + for (size_t off = 0; off < pcm.size(); off += chunk_n) { + const size_t n = std::min((size_t) chunk_n, pcm.size() - off); + queue.push(pcm.data() + off, n); + if (p.realtime) { + std::this_thread::sleep_for(std::chrono::milliseconds((1000 * (int) n) / WHISPER_SAMPLE_RATE)); + } + } + queue.close(); + + worker.join(); + + whisper_free_state(state); + whisper_free(ctx); + + return failed ? 2 : 0; +} diff --git a/include/whisper.h b/include/whisper.h index b5dcdb2917a..6945cfe2abe 100644 --- a/include/whisper.h +++ b/include/whisper.h @@ -457,6 +457,25 @@ extern "C" { WHISPER_SAMPLING_BEAM_SEARCH, // similar to OpenAI's BeamSearchDecoder }; + // Log-mel normalization mode (used by the resumable/streaming API) + enum whisper_mel_norm_mode { + // Normalize using the maximum log-mel value seen across all audio + // appended so far. This is the default. + // NOTE: it matches whisper_full() / batch behavior exactly only if the + // entire signal is appended before the first whisper_full_resumable() + // call. When decoding incrementally, early windows are normalized + // against the running max (not the whole-signal max), so a louder + // section that arrives later cannot retroactively change an already + // decoded window. For most speech this difference is negligible. + WHISPER_MEL_NORM_GLOBAL = 0, + + // Normalize each 30s encode window locally, using a reference level that + // is slew-rate limited relative to the previous window (asymmetric: + // fast attack when it gets louder, slow release so a brief silence does + // not amplify background noise). Intended for live transcription. + WHISPER_MEL_NORM_WINDOW = 1, + }; + // Text segment callback // Called on every newly generated text segment // Use the whisper_full_...() functions to obtain the text segments @@ -588,6 +607,17 @@ extern "C" { const char * vad_model_path; // Path to VAD model whisper_vad_params vad_params; + + // Log-mel normalization (used by the resumable/streaming API; ignored by whisper_full) + int mel_norm_mode; // enum whisper_mel_norm_mode; default WHISPER_MEL_NORM_GLOBAL (0) + // WINDOW mode reference-level envelope follower (in log10 power units): + // - attack (when a window gets louder) is instantaneous / free, so we never over-drive + // - release (when it gets quieter) decays exponentially in *audio time* toward the raw + // per-window peak, so a brief silence does not amplify background noise and a long + // steady source (e.g. a generator) that stops is forgotten only gradually + // The decay is fed the raw (un-clamped) per-window peak, so the history stays honest. + float mel_norm_half_life; // release half-life in seconds (audio time); <= 0 => instantaneous release + float mel_norm_max_drop; // optional cap on the reference drop rate (log10 power per second); <= 0 => unlimited }; // NOTE: this function allocates memory, and it is the responsibility of the caller to free the pointer - see whisper_free_context_params & whisper_free_params() @@ -613,6 +643,64 @@ extern "C" { const float * samples, int n_samples); + // ------------------------------------------------------------------------- + // Resumable / streaming transcription + // + // These functions let you feed audio incrementally and decode it in a way + // that is consistent with a single batch run: the seek position and the + // rolling text context are persisted in the state across calls, and audio + // is only ever decoded once (no sliding-window re-decoding / divergence). + // + // Typical usage from a worker thread that owns its own whisper_state: + // + // whisper_resumable_reset_with_state(ctx, state); + // while (recording) { + // whisper_append_audio_with_state(ctx, state, pcm, n); // as audio arrives + // int n_new = whisper_full_resumable_with_state(ctx, state, params, false); + // // ... consume the n_new newly produced segments ... + // } + // whisper_full_resumable_with_state(ctx, state, params, true); // flush the tail + // ------------------------------------------------------------------------- + + // Reset the resumable accumulation (clears accumulated audio, seek, context + // and results). Call this before starting a new stream/utterance. + WHISPER_API void whisper_resumable_reset_with_state( + struct whisper_context * ctx, + struct whisper_state * state); + WHISPER_API void whisper_resumable_reset(struct whisper_context * ctx); + + // Append PCM (16 kHz, mono, f32) to the resumable accumulation. The log-mel + // spectrogram is computed incrementally; the raw samples are not retained. + // Returns 0 on success, < 0 on error. + WHISPER_API int whisper_append_audio_with_state( + struct whisper_context * ctx, + struct whisper_state * state, + const float * samples, + int n_samples); + WHISPER_API int whisper_append_audio( + struct whisper_context * ctx, + const float * samples, + int n_samples); + + // Decode every complete 30s window currently available in the accumulated + // audio, resuming seek + context from the state. Newly produced segments are + // appended to the state's result list (so whisper_full_n_segments() returns + // the running total). If finalize is true, the trailing (< 30s) window is + // also decoded with zero-padding, matching batch behavior at end-of-audio. + // + // Returns the number of NEW segments produced by this call (>= 0), or < 0 on + // error. When finalize is false and less than 30s of undecoded audio is + // available, returns 0 without decoding a partial window ("need more audio"). + WHISPER_API int whisper_full_resumable_with_state( + struct whisper_context * ctx, + struct whisper_state * state, + struct whisper_full_params params, + bool finalize); + WHISPER_API int whisper_full_resumable( + struct whisper_context * ctx, + struct whisper_full_params params, + bool finalize); + // Split the input audio in chunks and process each chunk separately using whisper_full_with_state() // Result is stored in the default state of the context // Not thread safe if executed in parallel on the same context. diff --git a/src/whisper.cpp b/src/whisper.cpp index 5ffc70af00e..8d2e64e0369 100644 --- a/src/whisper.cpp +++ b/src/whisper.cpp @@ -920,6 +920,25 @@ struct whisper_state { // [EXPERIMENTAL] speed-up techniques int32_t exp_n_audio_ctx = 0; // 0 - use default + // [resumable / streaming] incremental decoding state + bool rs_active = false; // set while a resumable whisper_full_with_state call is running + bool rs_finalize = false; // process the trailing (< 30s) window with padding + int rs_seek = 0; // persisted seek position, in mel frames + int rs_n_frames = 0; // number of real mel frames accumulated (== mel.n_len_org) + int rs_n_pad = 0; // extra trailing "straddle" frames generated at finalize + std::vector rs_mel_raw; // raw (un-normalized) log-mel, frame-major: [frame*n_mel + band] + std::vector rs_pcm_buf; // unconsumed PCM samples for continuous framing + int rs_pcm_off = 0; // offset into rs_pcm_buf of the next not-yet-produced frame + std::vector rs_lead; // first samples buffered until the reflective pad can be built + bool rs_prepad_done = false;// reflective front pad has been emitted + int64_t rs_n_samples_in = 0; // total appended samples (diagnostics) + bool rs_lang_done = false; // language has been detected/fixed for this stream + float rs_global_max = -1e20f;// global raw max (WHISPER_MEL_NORM_GLOBAL) + float rs_prev_ref = 0.0f; // previous window reference level (WHISPER_MEL_NORM_WINDOW) + int rs_prev_ref_seek = 0; // seek (frames) at which rs_prev_ref was last updated + bool rs_have_ref = false; // rs_prev_ref initialized + bool rs_finalized = false; // a finalize() pass has run; further appends are rejected until reset + whisper_vad_context * vad_context = nullptr; struct vad_segment_info { @@ -6012,6 +6031,10 @@ struct whisper_full_params whisper_full_default_params(enum whisper_sampling_str /*.vad_model_path =*/ nullptr, /* vad_params =*/ whisper_vad_default_params(), + + /*.mel_norm_mode =*/ WHISPER_MEL_NORM_GLOBAL, + /*.mel_norm_half_life =*/ 3.0f, + /*.mel_norm_max_drop =*/ 0.0f, }; switch (strategy) { @@ -6035,6 +6058,8 @@ struct whisper_full_params whisper_full_default_params(enum whisper_sampling_str } // forward declarations +static void whisper_rs_fill_mel_window( + whisper_context * ctx, whisper_state * state, const whisper_full_params & params, int seek); static std::vector get_signal_energy(const float * signal, int n_samples, int n_samples_per_half_window); static void whisper_exp_compute_token_level_timestamps( struct whisper_context & ctx, @@ -6819,7 +6844,10 @@ int whisper_full_with_state( // clear old results auto & result_all = state->result_all; - result_all.clear(); + // in resumable mode the results accumulate across calls + if (!state->rs_active) { + result_all.clear(); + } if (n_samples > 0) { // compute log mel spectrogram @@ -7030,6 +7058,13 @@ int whisper_full_with_state( break; } + // [resumable] only decode windows that are fully backed by real audio; + // defer the trailing (< 30s) window until finalize. + if (state->rs_active && !state->rs_finalize && seek + WHISPER_CHUNK_SIZE*100 > seek_end) { + state->rs_seek = seek; + break; + } + if (params.encoder_begin_callback) { if (params.encoder_begin_callback(ctx, state, params.encoder_begin_callback_user_data) == false) { WHISPER_LOG_ERROR("%s: encoder_begin_callback returned false - aborting\n", __func__); @@ -7037,6 +7072,11 @@ int whisper_full_with_state( } } + // [resumable] materialize the normalized mel for the current window + if (state->rs_active) { + whisper_rs_fill_mel_window(ctx, state, params, seek); + } + // encode audio features starting at offset seek if (!whisper_encode_internal(*ctx, *state, seek, params.n_threads, params.abort_callback, params.abort_callback_user_data)) { WHISPER_LOG_ERROR("%s: failed to encode\n", __func__); @@ -7762,6 +7802,11 @@ int whisper_full_with_state( // update audio window seek += seek_delta; + // [resumable] persist the resume position + if (state->rs_active) { + state->rs_seek = seek; + } + WHISPER_LOG_DEBUG("seek = %d, seek_delta = %d\n", seek, seek_delta); } } @@ -7769,6 +7814,347 @@ int whisper_full_with_state( return 0; } +// ===================================================================================== +// Resumable / streaming transcription +// ===================================================================================== + +// Compute a single log-mel frame from `frame_size` contiguous samples and append the +// `n_mel` raw (un-normalized) log10 values to state->rs_mel_raw (frame-major layout). +static void whisper_rs_append_frame(whisper_context * ctx, whisper_state * state, const float * src) { + const whisper_filters & filters = ctx->model.filters; + + const int frame_size = WHISPER_N_FFT; // 400 + const int n_fft = filters.n_fft; // 201 (= 1 + frame_size/2) + const int n_mel = filters.n_mel; // 80 + + const float * hann = global_cache.hann_window; + + std::vector fft_in (frame_size * 2, 0.0f); + std::vector fft_out(frame_size * 2 * 2 * 2); + + // apply Hann window + for (int j = 0; j < frame_size; j++) { + fft_in[j] = hann[j] * src[j]; + } + + fft(fft_in.data(), frame_size, fft_out.data()); + + // modulus^2 of the complex bins + for (int j = 0; j < n_fft; j++) { + fft_out[j] = (fft_out[2 * j + 0] * fft_out[2 * j + 0] + fft_out[2 * j + 1] * fft_out[2 * j + 1]); + } + + // mel filterbank -> log10 (raw, before normalization) + const size_t base = state->rs_mel_raw.size(); + state->rs_mel_raw.resize(base + n_mel); + for (int j = 0; j < n_mel; j++) { + double sum = 0.0; + int k = 0; + for (k = 0; k < n_fft - 3; k += 4) { + sum += + fft_out[k + 0] * filters.data[j * n_fft + k + 0] + + fft_out[k + 1] * filters.data[j * n_fft + k + 1] + + fft_out[k + 2] * filters.data[j * n_fft + k + 2] + + fft_out[k + 3] * filters.data[j * n_fft + k + 3]; + } + for (; k < n_fft; k++) { + sum += fft_out[k] * filters.data[j * n_fft + k]; + } + sum = log10(std::max(sum, 1e-10)); + state->rs_mel_raw[base + j] = (float) sum; + + if ((float) sum > state->rs_global_max) { + state->rs_global_max = (float) sum; + } + } +} + +void whisper_resumable_reset_with_state(struct whisper_context * /*ctx*/, struct whisper_state * state) { + state->rs_active = false; + state->rs_finalize = false; + state->rs_seek = 0; + state->rs_n_frames = 0; + state->rs_n_pad = 0; + state->rs_mel_raw.clear(); + state->rs_pcm_buf.clear(); + state->rs_pcm_off = 0; + state->rs_lead.clear(); + state->rs_prepad_done = false; + state->rs_n_samples_in = 0; + state->rs_lang_done = false; + state->rs_global_max = -1e20f; + state->rs_prev_ref = 0.0f; + state->rs_prev_ref_seek = 0; + state->rs_have_ref = false; + state->rs_finalized = false; + + state->result_all.clear(); + state->prompt_past0.clear(); + state->prompt_past1.clear(); + + state->mel.data.clear(); + state->mel.n_len = 0; + state->mel.n_len_org = 0; +} + +void whisper_resumable_reset(struct whisper_context * ctx) { + whisper_resumable_reset_with_state(ctx, ctx->state); +} + +int whisper_append_audio_with_state( + struct whisper_context * ctx, + struct whisper_state * state, + const float * samples, + int n_samples) { + if (n_samples < 0 || (n_samples > 0 && samples == nullptr)) { + WHISPER_LOG_ERROR("%s: invalid arguments\n", __func__); + return -1; + } + + // appending after a finalize pass would interleave real frames behind the + // trailing zero-pad/straddle frames and desync the mel layout. The contract + // is finalize-then-reset; reject the misuse loudly instead of corrupting state. + if (state->rs_finalized) { + WHISPER_LOG_ERROR("%s: cannot append after finalize; call whisper_resumable_reset() first\n", __func__); + return -1; + } + + const int frame_size = WHISPER_N_FFT; // 400 + const int frame_step = WHISPER_HOP_LENGTH; // 160 + const int stage_2_pad = frame_size / 2; // 200 (reflective front pad) + + state->rs_n_samples_in += n_samples; + + if (!state->rs_prepad_done) { + // buffer the first samples until we can build the reflective front pad + state->rs_lead.insert(state->rs_lead.end(), samples, samples + n_samples); + if ((int) state->rs_lead.size() < stage_2_pad + 1) { + return 0; // wait for more audio + } + + // reflective pad: reverse of lead[1 .. stage_2_pad] -> lead[200],199,...,1 + for (int k = 0; k < stage_2_pad; k++) { + state->rs_pcm_buf.push_back(state->rs_lead[stage_2_pad - k]); + } + state->rs_pcm_buf.insert(state->rs_pcm_buf.end(), state->rs_lead.begin(), state->rs_lead.end()); + state->rs_lead.clear(); + state->rs_lead.shrink_to_fit(); + state->rs_prepad_done = true; + } else { + state->rs_pcm_buf.insert(state->rs_pcm_buf.end(), samples, samples + n_samples); + } + + // produce as many complete frames as the available samples allow + while ((int) state->rs_pcm_buf.size() - state->rs_pcm_off >= frame_size) { + whisper_rs_append_frame(ctx, state, state->rs_pcm_buf.data() + state->rs_pcm_off); + state->rs_pcm_off += frame_step; + state->rs_n_frames++; + } + + // compact the consumed prefix (keep the overlap needed for the next frame) + if (state->rs_pcm_off > 0) { + state->rs_pcm_buf.erase(state->rs_pcm_buf.begin(), state->rs_pcm_buf.begin() + state->rs_pcm_off); + state->rs_pcm_off = 0; + } + + return 0; +} + +int whisper_append_audio(struct whisper_context * ctx, const float * samples, int n_samples) { + return whisper_append_audio_with_state(ctx, ctx->state, samples, n_samples); +} + +// Fill the normalized, band-major mel slice [seek, seek+2*n_audio_ctx) of state->mel.data +// from the raw frame-major accumulation, using the configured normalization mode. +// Columns beyond the real audio (rs_n_frames) are filled with the normalized floor, +// matching how batch processing pads the trailing window with zero samples. +static void whisper_rs_fill_mel_window( + whisper_context * ctx, whisper_state * state, const whisper_full_params & params, int seek) { + const int n_mel = ctx->model.filters.n_mel; + const int n_len = state->mel.n_len; // stride (>= rs_n_frames, may include pad) + const int n_data = state->rs_n_frames + state->rs_n_pad; // frames with real mel data (incl. straddle) + const int win = 2 * whisper_n_audio_ctx(ctx); // 3000 frames (30s) + + const int i0 = std::min(seek, n_len); + const int i1 = std::min(seek + win, n_len); // end of the encode window (incl. pad) + const int i1_real = std::min(i1, n_data); // end of frames backed by real mel data + if (i1 <= i0) { + return; + } + + // reference level for this window + float ref; + if (params.mel_norm_mode == WHISPER_MEL_NORM_WINDOW) { + // raw, un-clamped peak of the current window (computed over the real frames only). + // this honest value is what feeds the release average. + float wmax = -1e20f; + for (int i = i0; i < i1_real; i++) { + const float * f = state->rs_mel_raw.data() + (size_t) i * n_mel; + for (int j = 0; j < n_mel; j++) { + if (f[j] > wmax) wmax = f[j]; + } + } + + if (!state->rs_have_ref) { + ref = wmax; // first window seeds the envelope + } else if (wmax >= state->rs_prev_ref) { + ref = wmax; // attack: instantaneous / free movement up + } else { + // release: exponential decay toward the (lower) current peak, parameterized by a + // half-life in audio seconds. dt is the audio time since the last reference update, + // i.e. the previous window's seek advance. This keeps the decay tied to elapsed + // audio time (not wall-clock or call frequency); note the stride itself is chosen by + // the decoder, so dt can vary from window to window. + const float dt = (seek - state->rs_prev_ref_seek) * 0.01f; // 1 frame = 10 ms + float decay = 0.0f; // <= 0 half-life => instantaneous release + if (params.mel_norm_half_life > 0.0f && dt > 0.0f) { + decay = powf(0.5f, dt / params.mel_norm_half_life); + } else if (params.mel_norm_half_life > 0.0f) { + decay = 1.0f; // no time elapsed => keep previous reference + } + ref = wmax + (state->rs_prev_ref - wmax) * decay; + // optional cap on the drop rate (log10 power per second) + if (params.mel_norm_max_drop > 0.0f && dt > 0.0f) { + ref = std::max(ref, state->rs_prev_ref - params.mel_norm_max_drop * dt); + } + } + state->rs_prev_ref = ref; + state->rs_prev_ref_seek = seek; + state->rs_have_ref = true; + } else { + ref = state->rs_global_max; + } + + const float floor_v = ref - 8.0f; + const float pad_val = (floor_v + 4.0f) / 4.0f; // normalized floor (== batch zero-sample padding) + + for (int j = 0; j < n_mel; j++) { + // real audio frames + for (int i = i0; i < i1_real; i++) { + float v = state->rs_mel_raw[(size_t) i * n_mel + j]; + if (v < floor_v) v = floor_v; + state->mel.data[(size_t) j * n_len + i] = (v + 4.0f) / 4.0f; + } + // trailing pad frames (finalize) + for (int i = std::max(i0, i1_real); i < i1; i++) { + state->mel.data[(size_t) j * n_len + i] = pad_val; + } + } +} + +int whisper_full_resumable_with_state( + struct whisper_context * ctx, + struct whisper_state * state, + struct whisper_full_params params, + bool finalize) { + const int win = 2 * whisper_n_audio_ctx(ctx); // 3000 frames (30s) + + // nothing to do yet: not enough audio for even a single complete window + if (!finalize && state->rs_n_frames - state->rs_seek < win) { + return 0; + } + if (state->rs_n_frames <= state->rs_seek) { + // on finalize, warn if audio was fed but too short to produce any frame + // (fewer than ~200 samples / 12.5ms never clears the reflective-pad gate) + if (finalize && state->rs_n_samples_in > 0 && state->rs_n_frames == 0) { + WHISPER_LOG_WARN("%s: audio too short (%lld samples); nothing transcribed\n", + __func__, (long long) state->rs_n_samples_in); + } + return 0; + } + + // [finalize] generate the trailing "straddle" frames whose 25ms window overlaps the + // real-audio / zero-pad boundary. Batch processing computes these from real audio + + // zero padding; we reproduce them by padding the unconsumed PCM tail with one frame of + // zeros. The remaining (fully-zero) pad frames are emitted as a constant by the fill. + if (finalize && state->rs_n_pad == 0 && + (int) state->rs_pcm_buf.size() - state->rs_pcm_off > 0) { + std::vector tail(state->rs_pcm_buf.begin() + state->rs_pcm_off, state->rs_pcm_buf.end()); + tail.insert(tail.end(), WHISPER_N_FFT, 0.0f); + int off = 0; + while ((int) tail.size() - off >= WHISPER_N_FFT) { + whisper_rs_append_frame(ctx, state, tail.data() + off); + off += WHISPER_HOP_LENGTH; + state->rs_n_pad++; + } + } + + // expose the accumulated frames as the (band-major) mel buffer for the encoder. + // the per-window normalized values are written lazily by whisper_rs_fill_mel_window(). + // on finalize we reserve an extra window of pad columns so the trailing (< 30s) + // window can be padded exactly like batch processing does (zero-sample padding). + const int pad = finalize ? win : 0; + state->mel.n_mel = ctx->model.filters.n_mel; + state->mel.n_len = state->rs_n_frames + pad; + state->mel.n_len_org = state->rs_n_frames; // seek_end: the decode loop stops at real audio + state->mel.data.assign((size_t) state->mel.n_mel * state->mel.n_len, 0.0f); + + // resume seek + context + params.offset_ms = state->rs_seek * 10; + params.duration_ms = 0; // up to rs_n_frames + if (state->rs_seek > 0) { + params.no_context = false; // keep the rolling context already in the state + } + + // resolve the language once, on the first processing call (the in-function + // auto-detect cannot be used here because the mel is materialized lazily) + if (!state->rs_lang_done) { + const bool want_detect = (params.language == nullptr || strlen(params.language) == 0 || + strcmp(params.language, "auto") == 0 || params.detect_language); + if (want_detect && whisper_is_multilingual(ctx)) { + whisper_full_params dp = params; + dp.mel_norm_mode = WHISPER_MEL_NORM_GLOBAL; // stable normalization for detection + // detection encodes the window at offset 0, so materialize the mel at 0 too. + // (language is resolved on the first processing call, where rs_seek == 0; filling + // at 0 keeps the fill/encode offsets consistent regardless of that assumption.) + whisper_rs_fill_mel_window(ctx, state, dp, 0); + + std::vector probs(whisper_lang_max_id() + 1, 0.0f); + const int lang_id = whisper_lang_auto_detect_with_state(ctx, state, 0, params.n_threads, probs.data()); + if (lang_id < 0) { + WHISPER_LOG_ERROR("%s: failed to auto-detect language\n", __func__); + return -3; + } + state->lang_id = lang_id; + WHISPER_LOG_INFO("%s: auto-detected language: %s\n", __func__, whisper_lang_str(lang_id)); + } else if (params.language != nullptr && strlen(params.language) > 0 && strcmp(params.language, "auto") != 0) { + state->lang_id = whisper_lang_id(params.language); + } else { + state->lang_id = whisper_lang_id("en"); + } + } + params.language = whisper_lang_str(state->lang_id); + params.detect_language = false; + + const int n_seg_before = (int) state->result_all.size(); + + state->rs_active = true; + state->rs_finalize = finalize; + + const int ret = whisper_full_with_state(ctx, state, params, nullptr, 0); + + state->rs_active = false; + state->rs_finalize = false; + + if (ret != 0) { + return ret < 0 ? ret : -1; + } + + state->rs_lang_done = true; + if (finalize) { + state->rs_finalized = true; // reject further appends until reset + } + + return (int) state->result_all.size() - n_seg_before; +} + +int whisper_full_resumable( + struct whisper_context * ctx, + struct whisper_full_params params, + bool finalize) { + return whisper_full_resumable_with_state(ctx, ctx->state, params, finalize); +} + int whisper_full( struct whisper_context * ctx, struct whisper_full_params params,