constrain auto-detect to a provided set of candidate languages

README.md

@@ -95,6 +95,14 @@ The command downloads the `base.en` model converted to custom `ggml` format and
 
 For detailed usage instructions, run: `./build/bin/whisper-cli -h`
 
+If you want auto-detect to choose from a known subset of languages, use `--language auto` together with `--language-candidates`. For example, to constrain detection to Indonesian or Arabic and prevent fallback to another languages such as Malay from being selected:
+
+```bash
+./build/bin/whisper-cli -m models/ggml-base.bin --language auto --language-candidates id,ar -f input.wav
+```
+
+If you already know the exact language, prefer a fixed language such as `--language id` instead of auto-detect.
+
 Note that the [whisper-cli](examples/cli) example currently runs only with 16-bit WAV files, so make sure to convert your input before running the tool.
 For example, you can use `ffmpeg` like this:
 

examples/cli/README.md

@@ -50,6 +50,7 @@ options:
   -nt,       --no-timestamps     [false  ] do not print timestamps
   -l LANG,   --language LANG     [en     ] spoken language ('auto' for auto-detect)
   -dl,       --detect-language   [false  ] exit after automatically detecting language
+             --language-candidates [      ] comma-separated language codes for constrained auto-detect
              --prompt PROMPT     [       ] initial prompt (max n_text_ctx/2 tokens)
   -m FNAME,  --model FNAME       [models/ggml-base.en.bin] model path
   -f FNAME,  --file FNAME        [       ] input audio file path

examples/cli/cli.cpp

@@ -115,6 +115,7 @@ struct whisper_params {
     bool carry_initial_prompt = false;
 
     std::string language  = "en";
+    std::vector<std::string> language_candidates = {};
     std::string prompt;
     std::string font_path = "/System/Library/Fonts/Supplemental/Courier New Bold.ttf";
     std::string model     = "models/ggml-base.en.bin";
@@ -162,6 +163,38 @@ static char * requires_value_error(const std::string & arg) {
     exit(0);
 }
 
+static std::vector<std::string> string_split(const std::string & input, char separator) {
+    std::vector<std::string> result;
+    size_t start = 0;
+
+    while (start <= input.size()) {
+        const size_t next = input.find(separator, start);
+        const std::string token = input.substr(start, next == std::string::npos ? std::string::npos : next - start);
+        result.push_back(token);
+
+        if (next == std::string::npos) {
+            break;
+        }
+
+        start = next + 1;
+    }
+
+    return result;
+}
+
+static std::string string_join(const std::vector<std::string> & values, const std::string & separator) {
+    std::string joined;
+
+    for (size_t i = 0; i < values.size(); ++i) {
+        if (i > 0) {
+            joined += separator;
+        }
+        joined += values[i];
+    }
+
+    return joined;
+}
+
 static bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
     if (const char * env_device = std::getenv("WHISPER_ARG_DEVICE")) {
         params.gpu_device = std::stoi(env_device);
@@ -225,6 +258,9 @@ static bool whisper_params_parse(int argc, char ** argv, whisper_params & params
         else if (arg == "-nt"   || arg == "--no-timestamps")        { params.no_timestamps   = true; }
         else if (arg == "-l"    || arg == "--language")             { params.language        = whisper_param_turn_lowercase(ARGV_NEXT); }
         else if (arg == "-dl"   || arg == "--detect-language")      { params.detect_language = true; }
+        else if (                  arg == "--language-candidates")  {
+            params.language_candidates = string_split(whisper_param_turn_lowercase(ARGV_NEXT), ',');
+        }
         else if (                  arg == "--prompt")               { params.prompt          = ARGV_NEXT; }
         else if (                  arg == "--carry-initial-prompt") { params.carry_initial_prompt = true; }
         else if (arg == "-m"    || arg == "--model")                { params.model           = ARGV_NEXT; }
@@ -307,6 +343,7 @@ static void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params
     fprintf(stderr, "  -nt,       --no-timestamps        [%-7s] do not print timestamps\n",                        params.no_timestamps ? "true" : "false");
     fprintf(stderr, "  -l LANG,   --language LANG        [%-7s] spoken language ('auto' for auto-detect)\n",       params.language.c_str());
     fprintf(stderr, "  -dl,       --detect-language      [%-7s] exit after automatically detecting language\n",    params.detect_language ? "true" : "false");
+    fprintf(stderr, "             --language-candidates  [%-7s] comma-separated language codes for constrained auto-detect\n", string_join(params.language_candidates, ",").c_str());
     fprintf(stderr, "             --prompt PROMPT        [%-7s] initial prompt (max n_text_ctx/2 tokens)\n",       params.prompt.c_str());
     fprintf(stderr, "             --carry-initial-prompt [%-7s] always prepend initial prompt\n",                  params.carry_initial_prompt ? "true" : "false");
     fprintf(stderr, "  -m FNAME,  --model FNAME          [%-7s] model path\n",                                     params.model.c_str());
@@ -715,6 +752,24 @@ static void output_json(
         end_value(end);
     };
 
+    auto value_arr_s = [&](const char * name, const std::vector<std::string> & vals, bool end) {
+        doindent();
+        fout << "\"" << name << "\": [";
+        if (!vals.empty()) {
+            fout << "\n";
+            indent++;
+            for (size_t i = 0; i < vals.size(); ++i) {
+                doindent();
+                char * val_escaped = escape_double_quotes_and_backslashes(vals[i].c_str());
+                fout << "\"" << val_escaped << "\"" << (i + 1 == vals.size() ? "\n" : ",\n");
+                free(val_escaped);
+            }
+            indent--;
+            doindent();
+        }
+        fout << "]" << (end ? "\n" : ",\n");
+    };
+
     auto times_o = [&](int64_t t0, int64_t t1, bool end) {
         start_obj("timestamps");
         value_s("from", to_timestamp(t0, true).c_str(), false);
@@ -750,6 +805,7 @@ static void output_json(
         start_obj("params");
             value_s("model", params.model.c_str(), false);
             value_s("language", params.language.c_str(), false);
+            value_arr_s("language_candidates", params.language_candidates, false);
             value_b("translate", params.translate, true);
         end_obj(false);
         start_obj("result");
@@ -1058,6 +1114,20 @@ int main(int argc, char ** argv) {
         exit(0);
     }
 
+    if (!params.language_candidates.empty() && params.language != "auto" && !params.detect_language) {
+        fprintf(stderr, "error: --language-candidates requires --language auto or --detect-language\n");
+        whisper_print_usage(argc, argv, params);
+        exit(0);
+    }
+
+    for (const auto & candidate : params.language_candidates) {
+        if (candidate.empty() || whisper_lang_id(candidate.c_str()) == -1) {
+            fprintf(stderr, "error: unknown language candidate '%s'\n", candidate.c_str());
+            whisper_print_usage(argc, argv, params);
+            exit(0);
+        }
+    }
+
     if (params.diarize && params.tinydiarize) {
         fprintf(stderr, "error: cannot use both --diarize and --tinydiarize\n");
         whisper_print_usage(argc, argv, params);
@@ -1105,6 +1175,12 @@ int main(int argc, char ** argv) {
         return 3;
     }
 
+    if (!params.language_candidates.empty() && !whisper_is_multilingual(ctx)) {
+        fprintf(stderr, "error: --language-candidates requires a multilingual model\n");
+        whisper_free(ctx);
+        return 3;
+    }
+
     // initialize openvino encoder. this has no effect on whisper.cpp builds that don't have OpenVINO configured
     whisper_ctx_init_openvino_encoder(ctx, nullptr, params.openvino_encode_device.c_str(), nullptr);
 
@@ -1202,6 +1278,7 @@ int main(int argc, char ** argv) {
         }
 
         if (!params.no_prints) {
+            const std::string candidate_summary = string_join(params.language_candidates, ",");
             // print system information
             fprintf(stderr, "\n");
             fprintf(stderr, "system_info: n_threads = %d / %d | %s\n",
@@ -1216,6 +1293,9 @@ int main(int argc, char ** argv) {
                     params.translate ? "translate" : "transcribe",
                     params.tinydiarize ? "tdrz = 1, " : "",
                     params.no_timestamps ? 0 : 1);
+            if (!candidate_summary.empty()) {
+                fprintf(stderr, "%s: constrained auto-detect candidates = %s\n", __func__, candidate_summary.c_str());
+            }
 
             if (params.print_colors) {
                 fprintf(stderr, "%s: color scheme: red (low confidence), yellow (medium), green (high confidence)\n", __func__);
@@ -1228,6 +1308,11 @@ int main(int argc, char ** argv) {
         // run the inference
         {
             whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
+            std::vector<const char *> language_candidates;
+            language_candidates.reserve(params.language_candidates.size());
+            for (const auto & candidate : params.language_candidates) {
+                language_candidates.push_back(candidate.c_str());
+            }
 
             const bool use_grammar = (!params.grammar_parsed.rules.empty() && !params.grammar_rule.empty());
             wparams.strategy = (params.beam_size > 1 || use_grammar) ? WHISPER_SAMPLING_BEAM_SEARCH : WHISPER_SAMPLING_GREEDY;
@@ -1239,6 +1324,8 @@ int main(int argc, char ** argv) {
             wparams.translate        = params.translate;
             wparams.language         = params.language.c_str();
             wparams.detect_language  = params.detect_language;
+            wparams.language_candidates = language_candidates.empty() ? nullptr : language_candidates.data();
+            wparams.n_language_candidates = language_candidates.size();
             wparams.n_threads        = params.n_threads;
             wparams.n_max_text_ctx   = params.max_context >= 0 ? params.max_context : wparams.n_max_text_ctx;
             wparams.offset_ms        = params.offset_t_ms;

include/whisper.h

@@ -381,13 +381,30 @@ extern "C" {
                                int   n_threads,
                              float * lang_probs);
 
+    WHISPER_API int whisper_lang_auto_detect_candidates(
+            struct whisper_context * ctx,
+                               int   offset_ms,
+                               int   n_threads,
+                     const char * const * language_candidates,
+                               int   n_language_candidates,
+                             float * lang_probs);
+
     WHISPER_API int whisper_lang_auto_detect_with_state(
             struct whisper_context * ctx,
               struct whisper_state * state,
                                int   offset_ms,
                                int   n_threads,
                              float * lang_probs);
 
+    WHISPER_API int whisper_lang_auto_detect_with_state_candidates(
+            struct whisper_context * ctx,
+              struct whisper_state * state,
+                               int   offset_ms,
+                               int   n_threads,
+                     const char * const * language_candidates,
+                               int   n_language_candidates,
+                             float * lang_probs);
+
     WHISPER_API int whisper_n_len           (struct whisper_context * ctx); // mel length
     WHISPER_API int whisper_n_len_from_state(struct whisper_state * state); // mel length
     WHISPER_API int whisper_n_vocab         (struct whisper_context * ctx);
@@ -532,6 +549,8 @@ extern "C" {
         // for auto-detection, set to nullptr, "" or "auto"
         const char * language;
         bool detect_language;
+        const char * const * language_candidates;
+        int n_language_candidates;
 
         // common decoding parameters:
         bool suppress_blank; // ref: https://github.com/openai/whisper/blob/f82bc59f5ea234d4b97fb2860842ed38519f7e65/whisper/decoding.py#L89