diff --git a/README.md b/README.md index 050a35be21c..d25270fae9c 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,14 @@ The command downloads the `base.en` model converted to custom `ggml` format and For detailed usage instructions, run: `./build/bin/whisper-cli -h` +If you want auto-detect to choose from a known subset of languages, use `--language auto` together with `--language-candidates`. For example, to constrain detection to Indonesian or Arabic and prevent fallback to another languages such as Malay from being selected: + +```bash +./build/bin/whisper-cli -m models/ggml-base.bin --language auto --language-candidates id,ar -f input.wav +``` + +If you already know the exact language, prefer a fixed language such as `--language id` instead of auto-detect. + Note that the [whisper-cli](examples/cli) example currently runs only with 16-bit WAV files, so make sure to convert your input before running the tool. For example, you can use `ffmpeg` like this: diff --git a/examples/cli/README.md b/examples/cli/README.md index 65285c3cb66..f25904d6f50 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -50,6 +50,7 @@ options: -nt, --no-timestamps [false ] do not print timestamps -l LANG, --language LANG [en ] spoken language ('auto' for auto-detect) -dl, --detect-language [false ] exit after automatically detecting language + --language-candidates [ ] comma-separated language codes for constrained auto-detect --prompt PROMPT [ ] initial prompt (max n_text_ctx/2 tokens) -m FNAME, --model FNAME [models/ggml-base.en.bin] model path -f FNAME, --file FNAME [ ] input audio file path diff --git a/examples/cli/cli.cpp b/examples/cli/cli.cpp index 55cd71b4e55..4a9c108508f 100644 --- a/examples/cli/cli.cpp +++ b/examples/cli/cli.cpp @@ -115,6 +115,7 @@ struct whisper_params { bool carry_initial_prompt = false; std::string language = "en"; + std::vector language_candidates = {}; std::string prompt; std::string font_path = "/System/Library/Fonts/Supplemental/Courier New Bold.ttf"; std::string model = "models/ggml-base.en.bin"; @@ -162,6 +163,38 @@ static char * requires_value_error(const std::string & arg) { exit(0); } +static std::vector string_split(const std::string & input, char separator) { + std::vector result; + size_t start = 0; + + while (start <= input.size()) { + const size_t next = input.find(separator, start); + const std::string token = input.substr(start, next == std::string::npos ? std::string::npos : next - start); + result.push_back(token); + + if (next == std::string::npos) { + break; + } + + start = next + 1; + } + + return result; +} + +static std::string string_join(const std::vector & values, const std::string & separator) { + std::string joined; + + for (size_t i = 0; i < values.size(); ++i) { + if (i > 0) { + joined += separator; + } + joined += values[i]; + } + + return joined; +} + static bool whisper_params_parse(int argc, char ** argv, whisper_params & params) { if (const char * env_device = std::getenv("WHISPER_ARG_DEVICE")) { params.gpu_device = std::stoi(env_device); @@ -225,6 +258,9 @@ static bool whisper_params_parse(int argc, char ** argv, whisper_params & params else if (arg == "-nt" || arg == "--no-timestamps") { params.no_timestamps = true; } else if (arg == "-l" || arg == "--language") { params.language = whisper_param_turn_lowercase(ARGV_NEXT); } else if (arg == "-dl" || arg == "--detect-language") { params.detect_language = true; } + else if ( arg == "--language-candidates") { + params.language_candidates = string_split(whisper_param_turn_lowercase(ARGV_NEXT), ','); + } else if ( arg == "--prompt") { params.prompt = ARGV_NEXT; } else if ( arg == "--carry-initial-prompt") { params.carry_initial_prompt = true; } else if (arg == "-m" || arg == "--model") { params.model = ARGV_NEXT; } @@ -307,6 +343,7 @@ static void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params fprintf(stderr, " -nt, --no-timestamps [%-7s] do not print timestamps\n", params.no_timestamps ? "true" : "false"); fprintf(stderr, " -l LANG, --language LANG [%-7s] spoken language ('auto' for auto-detect)\n", params.language.c_str()); fprintf(stderr, " -dl, --detect-language [%-7s] exit after automatically detecting language\n", params.detect_language ? "true" : "false"); + fprintf(stderr, " --language-candidates [%-7s] comma-separated language codes for constrained auto-detect\n", string_join(params.language_candidates, ",").c_str()); fprintf(stderr, " --prompt PROMPT [%-7s] initial prompt (max n_text_ctx/2 tokens)\n", params.prompt.c_str()); fprintf(stderr, " --carry-initial-prompt [%-7s] always prepend initial prompt\n", params.carry_initial_prompt ? "true" : "false"); fprintf(stderr, " -m FNAME, --model FNAME [%-7s] model path\n", params.model.c_str()); @@ -715,6 +752,24 @@ static void output_json( end_value(end); }; + auto value_arr_s = [&](const char * name, const std::vector & vals, bool end) { + doindent(); + fout << "\"" << name << "\": ["; + if (!vals.empty()) { + fout << "\n"; + indent++; + for (size_t i = 0; i < vals.size(); ++i) { + doindent(); + char * val_escaped = escape_double_quotes_and_backslashes(vals[i].c_str()); + fout << "\"" << val_escaped << "\"" << (i + 1 == vals.size() ? "\n" : ",\n"); + free(val_escaped); + } + indent--; + doindent(); + } + fout << "]" << (end ? "\n" : ",\n"); + }; + auto times_o = [&](int64_t t0, int64_t t1, bool end) { start_obj("timestamps"); value_s("from", to_timestamp(t0, true).c_str(), false); @@ -750,6 +805,7 @@ static void output_json( start_obj("params"); value_s("model", params.model.c_str(), false); value_s("language", params.language.c_str(), false); + value_arr_s("language_candidates", params.language_candidates, false); value_b("translate", params.translate, true); end_obj(false); start_obj("result"); @@ -1058,6 +1114,20 @@ int main(int argc, char ** argv) { exit(0); } + if (!params.language_candidates.empty() && params.language != "auto" && !params.detect_language) { + fprintf(stderr, "error: --language-candidates requires --language auto or --detect-language\n"); + whisper_print_usage(argc, argv, params); + exit(0); + } + + for (const auto & candidate : params.language_candidates) { + if (candidate.empty() || whisper_lang_id(candidate.c_str()) == -1) { + fprintf(stderr, "error: unknown language candidate '%s'\n", candidate.c_str()); + whisper_print_usage(argc, argv, params); + exit(0); + } + } + if (params.diarize && params.tinydiarize) { fprintf(stderr, "error: cannot use both --diarize and --tinydiarize\n"); whisper_print_usage(argc, argv, params); @@ -1105,6 +1175,12 @@ int main(int argc, char ** argv) { return 3; } + if (!params.language_candidates.empty() && !whisper_is_multilingual(ctx)) { + fprintf(stderr, "error: --language-candidates requires a multilingual model\n"); + whisper_free(ctx); + return 3; + } + // initialize openvino encoder. this has no effect on whisper.cpp builds that don't have OpenVINO configured whisper_ctx_init_openvino_encoder(ctx, nullptr, params.openvino_encode_device.c_str(), nullptr); @@ -1202,6 +1278,7 @@ int main(int argc, char ** argv) { } if (!params.no_prints) { + const std::string candidate_summary = string_join(params.language_candidates, ","); // print system information fprintf(stderr, "\n"); fprintf(stderr, "system_info: n_threads = %d / %d | %s\n", @@ -1216,6 +1293,9 @@ int main(int argc, char ** argv) { params.translate ? "translate" : "transcribe", params.tinydiarize ? "tdrz = 1, " : "", params.no_timestamps ? 0 : 1); + if (!candidate_summary.empty()) { + fprintf(stderr, "%s: constrained auto-detect candidates = %s\n", __func__, candidate_summary.c_str()); + } if (params.print_colors) { fprintf(stderr, "%s: color scheme: red (low confidence), yellow (medium), green (high confidence)\n", __func__); @@ -1228,6 +1308,11 @@ int main(int argc, char ** argv) { // run the inference { whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY); + std::vector language_candidates; + language_candidates.reserve(params.language_candidates.size()); + for (const auto & candidate : params.language_candidates) { + language_candidates.push_back(candidate.c_str()); + } const bool use_grammar = (!params.grammar_parsed.rules.empty() && !params.grammar_rule.empty()); wparams.strategy = (params.beam_size > 1 || use_grammar) ? WHISPER_SAMPLING_BEAM_SEARCH : WHISPER_SAMPLING_GREEDY; @@ -1239,6 +1324,8 @@ int main(int argc, char ** argv) { wparams.translate = params.translate; wparams.language = params.language.c_str(); wparams.detect_language = params.detect_language; + wparams.language_candidates = language_candidates.empty() ? nullptr : language_candidates.data(); + wparams.n_language_candidates = language_candidates.size(); wparams.n_threads = params.n_threads; wparams.n_max_text_ctx = params.max_context >= 0 ? params.max_context : wparams.n_max_text_ctx; wparams.offset_ms = params.offset_t_ms; diff --git a/include/whisper.h b/include/whisper.h index b5dcdb2917a..2b81f9817d0 100644 --- a/include/whisper.h +++ b/include/whisper.h @@ -381,6 +381,14 @@ extern "C" { int n_threads, float * lang_probs); + WHISPER_API int whisper_lang_auto_detect_candidates( + struct whisper_context * ctx, + int offset_ms, + int n_threads, + const char * const * language_candidates, + int n_language_candidates, + float * lang_probs); + WHISPER_API int whisper_lang_auto_detect_with_state( struct whisper_context * ctx, struct whisper_state * state, @@ -388,6 +396,15 @@ extern "C" { int n_threads, float * lang_probs); + WHISPER_API int whisper_lang_auto_detect_with_state_candidates( + struct whisper_context * ctx, + struct whisper_state * state, + int offset_ms, + int n_threads, + const char * const * language_candidates, + int n_language_candidates, + float * lang_probs); + WHISPER_API int whisper_n_len (struct whisper_context * ctx); // mel length WHISPER_API int whisper_n_len_from_state(struct whisper_state * state); // mel length WHISPER_API int whisper_n_vocab (struct whisper_context * ctx); @@ -532,6 +549,8 @@ extern "C" { // for auto-detection, set to nullptr, "" or "auto" const char * language; bool detect_language; + const char * const * language_candidates; + int n_language_candidates; // common decoding parameters: bool suppress_blank; // ref: https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/decoding.py#L89 diff --git a/src/whisper.cpp b/src/whisper.cpp index 0fe29a4541e..9fbfca507d3 100644 --- a/src/whisper.cpp +++ b/src/whisper.cpp @@ -3996,6 +3996,56 @@ int whisper_lang_id(const char * lang) { return g_lang.at(lang).first; } +int whisper_lang_candidates_to_ids( + const char * func_name, + const char * const * language_candidates, + int n_language_candidates, + std::vector & lang_ids) { + lang_ids.clear(); + + if (n_language_candidates < 0) { + WHISPER_LOG_ERROR("%s: invalid candidate count %d\n", func_name, n_language_candidates); + return -4; + } + + if (n_language_candidates == 0) { + for (const auto & kv : g_lang) { + lang_ids.push_back(kv.second.first); + } + return 0; + } + + if (language_candidates == nullptr) { + WHISPER_LOG_ERROR("%s: language_candidates is null with n_language_candidates = %d\n", func_name, n_language_candidates); + return -4; + } + + for (int i = 0; i < n_language_candidates; ++i) { + const char * candidate = language_candidates[i]; + if (candidate == nullptr) { + WHISPER_LOG_ERROR("%s: language_candidates[%d] is null\n", func_name, i); + return -4; + } + + const int lang_id = whisper_lang_id(candidate); + if (lang_id < 0) { + WHISPER_LOG_ERROR("%s: invalid language candidate '%s'\n", func_name, candidate); + return -4; + } + + if (std::find(lang_ids.begin(), lang_ids.end(), lang_id) == lang_ids.end()) { + lang_ids.push_back(lang_id); + } + } + + if (lang_ids.empty()) { + WHISPER_LOG_ERROR("%s: no valid language candidates provided\n", func_name); + return -4; + } + + return 0; +} + const char * whisper_lang_str(int id) { for (const auto & kv : g_lang) { if (kv.second.first == id) { @@ -4018,43 +4068,58 @@ const char * whisper_lang_str_full(int id) { return nullptr; } -int whisper_lang_auto_detect_with_state( +static int whisper_lang_auto_detect_impl( + const char * func_name, struct whisper_context * ctx, struct whisper_state * state, int offset_ms, int n_threads, + const char * const * language_candidates, + int n_language_candidates, float * lang_probs) { + std::vector lang_ids; + const int candidates_result = whisper_lang_candidates_to_ids(func_name, language_candidates, n_language_candidates, lang_ids); + if (candidates_result != 0) { + return candidates_result; + } + const int seek = offset_ms/10; + if (lang_probs) { + const int n_langs = whisper_lang_max_id() + 1; + std::fill(lang_probs, lang_probs + n_langs, 0.0f); + } + if (seek < 0) { - WHISPER_LOG_ERROR("%s: offset %dms is before the start of the audio\n", __func__, offset_ms); + WHISPER_LOG_ERROR("%s: offset %dms is before the start of the audio\n", func_name, offset_ms); return -1; } if (seek >= state->mel.n_len_org) { - WHISPER_LOG_ERROR("%s: offset %dms is past the end of the audio (%dms)\n", __func__, offset_ms, state->mel.n_len_org*10); + WHISPER_LOG_ERROR("%s: offset %dms is past the end of the audio (%dms)\n", func_name, offset_ms, state->mel.n_len_org*10); return -2; } // run the encoder if (whisper_encode_with_state(ctx, state, seek, n_threads) != 0) { - WHISPER_LOG_ERROR("%s: failed to encode\n", __func__); + WHISPER_LOG_ERROR("%s: failed to encode\n", func_name); return -6; } const std::vector prompt = { whisper_token_sot(ctx) }; if (whisper_decode_with_state(ctx, state, prompt.data(), prompt.size(), 0, n_threads) != 0) { - WHISPER_LOG_ERROR("%s: failed to decode\n", __func__); + WHISPER_LOG_ERROR("%s: failed to decode\n", func_name); return -7; } auto & logits_id = state->decoders[0].logits_id; logits_id.clear(); + logits_id.reserve(lang_ids.size()); - for (const auto & kv : g_lang) { - const auto token_lang = whisper_token_lang(ctx, kv.second.first); - logits_id.emplace_back(state->logits[token_lang], kv.second.first); + for (const int lang_id : lang_ids) { + const auto token_lang = whisper_token_lang(ctx, lang_id); + logits_id.emplace_back(state->logits[token_lang], lang_id); } // sort descending @@ -4086,7 +4151,7 @@ int whisper_lang_auto_detect_with_state( lang_probs[prob.second] = prob.first; } - //printf("%s: lang %2d (%3s): %f\n", __func__, prob.second, whisper_lang_str(prob.second), prob.first); + //printf("%s: lang %2d (%3s): %f\n", func_name, prob.second, whisper_lang_str(prob.second), prob.first); } } @@ -4101,6 +4166,36 @@ int whisper_lang_auto_detect( return whisper_lang_auto_detect_with_state(ctx, ctx->state, offset_ms, n_threads, lang_probs); } +int whisper_lang_auto_detect_candidates( + struct whisper_context * ctx, + int offset_ms, + int n_threads, + const char * const * language_candidates, + int n_language_candidates, + float * lang_probs) { + return whisper_lang_auto_detect_with_state_candidates(ctx, ctx->state, offset_ms, n_threads, language_candidates, n_language_candidates, lang_probs); +} + +int whisper_lang_auto_detect_with_state( + struct whisper_context * ctx, + struct whisper_state * state, + int offset_ms, + int n_threads, + float * lang_probs) { + return whisper_lang_auto_detect_impl(__func__, ctx, state, offset_ms, n_threads, nullptr, 0, lang_probs); +} + +int whisper_lang_auto_detect_with_state_candidates( + struct whisper_context * ctx, + struct whisper_state * state, + int offset_ms, + int n_threads, + const char * const * language_candidates, + int n_language_candidates, + float * lang_probs) { + return whisper_lang_auto_detect_impl(__func__, ctx, state, offset_ms, n_threads, language_candidates, n_language_candidates, lang_probs); +} + int whisper_model_n_vocab(struct whisper_context * ctx) { return ctx->model.hparams.n_vocab; } @@ -5951,6 +6046,8 @@ struct whisper_full_params whisper_full_default_params(enum whisper_sampling_str /*.language =*/ "en", /*.detect_language =*/ false, + /*.language_candidates =*/ nullptr, + /*.n_language_candidates =*/ 0, /*.suppress_blank =*/ true, /*.suppress_nst =*/ false, @@ -6819,7 +6916,11 @@ int whisper_full_with_state( if (params.language == nullptr || strlen(params.language) == 0 || strcmp(params.language, "auto") == 0 || params.detect_language) { std::vector probs(whisper_lang_max_id() + 1, 0.0f); - const auto lang_id = whisper_lang_auto_detect_with_state(ctx, state, 0, params.n_threads, probs.data()); + const auto lang_id = params.n_language_candidates > 0 + ? whisper_lang_auto_detect_with_state_candidates( + ctx, state, 0, params.n_threads, + params.language_candidates, params.n_language_candidates, probs.data()) + : whisper_lang_auto_detect_with_state(ctx, state, 0, params.n_threads, probs.data()); if (lang_id < 0) { WHISPER_LOG_ERROR("%s: failed to auto-detect language\n", __func__); return -3; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 09e77ea89c2..95418960edc 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -78,6 +78,24 @@ add_test(NAME ${TEST_TARGET} -f ${PROJECT_SOURCE_DIR}/samples/jfk.wav) set_tests_properties(${TEST_TARGET} PROPERTIES LABELS "large") +set(TEST_TARGET test-whisper-cli-detect-language-candidates) +add_test(NAME ${TEST_TARGET} + COMMAND $ + -m ${PROJECT_SOURCE_DIR}/models/for-tests-ggml-base.bin + -ng -dl --language-candidates en,fr + -f ${PROJECT_SOURCE_DIR}/samples/jfk.wav) +set_tests_properties(${TEST_TARGET} PROPERTIES LABELS "base;gh") + +set(TEST_TARGET test-whisper-cli-detect-language-candidates-nonmultilingual) +add_test(NAME ${TEST_TARGET} + COMMAND $ + -m ${PROJECT_SOURCE_DIR}/models/for-tests-ggml-base.en.bin + -ng -dl --language-candidates en,fr + -f ${PROJECT_SOURCE_DIR}/samples/jfk.wav) +set_tests_properties(${TEST_TARGET} PROPERTIES + LABELS "base;en" + WILL_FAIL TRUE) + if (WHISPER_FFMPEG) set(TEST_TARGET test-whisper-cli-tiny-mp3) # Check with reviewers: any way to check the output transcription via ctest (diff, ...)? @@ -110,3 +128,14 @@ target_compile_definitions(${VAD_TEST} PRIVATE SAMPLE_PATH="${PROJECT_SOURCE_DIR}/samples/jfk.wav") add_test(NAME ${VAD_TEST} COMMAND ${VAD_TEST}) set_tests_properties(${VAD_TEST} PROPERTIES LABELS "base;en") + +set(LANG_TEST test-lang-auto-detect) +add_executable(${LANG_TEST} ${LANG_TEST}.cpp) +target_include_directories(${LANG_TEST} PRIVATE ../include ../ggml/include ../examples) +target_link_libraries(${LANG_TEST} PRIVATE common) +target_compile_definitions(${LANG_TEST} PRIVATE + WHISPER_MODEL_PATH="${PROJECT_SOURCE_DIR}/models/for-tests-ggml-base.bin" + WHISPER_MODEL_EN_PATH="${PROJECT_SOURCE_DIR}/models/for-tests-ggml-base.en.bin" + SAMPLE_PATH="${PROJECT_SOURCE_DIR}/samples/jfk.wav") +add_test(NAME ${LANG_TEST} COMMAND ${LANG_TEST}) +set_tests_properties(${LANG_TEST} PROPERTIES LABELS "base;en;unit") diff --git a/tests/test-lang-auto-detect.cpp b/tests/test-lang-auto-detect.cpp new file mode 100644 index 00000000000..198eec71806 --- /dev/null +++ b/tests/test-lang-auto-detect.cpp @@ -0,0 +1,66 @@ +#include "whisper.h" +#include "common-whisper.h" + +#include +#include +#include + +static void assert_only_candidates_have_probability(const std::vector & probs, int lang_a, int lang_b) { + for (int i = 0; i < (int) probs.size(); ++i) { + if (i == lang_a || i == lang_b) { + assert(probs[i] >= 0.0f); + } else { + assert(probs[i] == 0.0f); + } + } +} + +int main() { + std::vector pcmf32; + std::vector> pcmf32s; + assert(read_audio_data(SAMPLE_PATH, pcmf32, pcmf32s, false)); + + struct whisper_context_params cparams = whisper_context_default_params(); + cparams.use_gpu = false; + + struct whisper_context * wctx = whisper_init_from_file_with_params(WHISPER_MODEL_PATH, cparams); + assert(wctx != nullptr); + assert(whisper_is_multilingual(wctx)); + assert(whisper_pcm_to_mel(wctx, pcmf32.data(), pcmf32.size(), 1) == 0); + + const char * candidates[] = {"fr", "en", "en"}; + std::vector probs(whisper_lang_max_id() + 1, -1.0f); + + const int lang_id = whisper_lang_auto_detect_candidates(wctx, 0, 1, candidates, 3, probs.data()); + assert(lang_id == whisper_lang_id("en")); + assert(probs[whisper_lang_id("en")] > 0.0f); + assert(probs[whisper_lang_id("fr")] > 0.0f); + assert_only_candidates_have_probability(probs, whisper_lang_id("en"), whisper_lang_id("fr")); + + const char * invalid_candidates[] = {"zz"}; + assert(whisper_lang_auto_detect_candidates(wctx, 0, 1, invalid_candidates, 1, nullptr) == -4); + + const char * null_candidate = nullptr; + assert(whisper_lang_auto_detect_candidates(wctx, 0, 1, &null_candidate, 1, nullptr) == -4); + + struct whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY); + wparams.language = "auto"; + wparams.detect_language = true; + wparams.language_candidates = candidates; + wparams.n_language_candidates = 3; + + assert(whisper_full(wctx, wparams, pcmf32.data(), pcmf32.size()) == 0); + assert(whisper_full_lang_id(wctx) == whisper_lang_id("en")); + + whisper_free(wctx); + + struct whisper_context * wctx_en = whisper_init_from_file_with_params(WHISPER_MODEL_EN_PATH, cparams); + assert(wctx_en != nullptr); + assert(!whisper_is_multilingual(wctx_en)); + assert(whisper_pcm_to_mel(wctx_en, pcmf32.data(), pcmf32.size(), 1) == 0); + assert(whisper_lang_auto_detect_candidates(wctx_en, 0, 1, candidates, 2, nullptr) == -5); + + whisper_free(wctx_en); + + return 0; +}