diff --git a/server/src/server/sse_emitter.cpp b/server/src/server/sse_emitter.cpp index f4d7bf1df..1c9707e78 100644 --- a/server/src/server/sse_emitter.cpp +++ b/server/src/server/sse_emitter.cpp @@ -13,13 +13,6 @@ namespace dflash::common { static const char THINK_OPEN[] = ""; static const char THINK_CLOSE[] = ""; -static const char TOOL_OPEN[] = ""; -static const char FUNCTION_OPEN[] = "KV` with the -// wrapper as special tokens that detokenization strips, so the -// visible trigger is and the tool name sits immediately before it. -static const char ARG_KEY_OPEN[] = ""; static constexpr size_t THINK_OPEN_LEN = 7; static constexpr size_t THINK_CLOSE_LEN = 8; @@ -38,34 +31,6 @@ static bool starts_with_potential_bare_json_tool(const std::string & text, return first != std::string::npos && text[first] == '{'; } -static bool find_tool_start(const std::string & text, size_t & pos) { - size_t idx = text.find('<'); - while (idx != std::string::npos) { - if (text.compare(idx, sizeof(TOOL_OPEN) - 1, TOOL_OPEN) == 0 || - text.compare(idx, sizeof(FUNCTION_OPEN) - 1, FUNCTION_OPEN) == 0 || - text.compare(idx, sizeof(TOOL_CODE_OPEN) - 1, TOOL_CODE_OPEN) == 0) { - pos = idx; - return true; - } - if (text.compare(idx, sizeof(ARG_KEY_OPEN) - 1, ARG_KEY_OPEN) == 0) { - // Rewind over the tool name so it lands in the tool buffer - // (the parser extracts it from just before ). - size_t start = idx; - auto is_ident = [](char c) { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || c == '_' || c == '-'; - }; - while (start > 0 && is_ident(text[start - 1])) start--; - if (start < idx) { // require a non-empty name - pos = start; - return true; - } - } - idx = text.find('<', idx + 1); - } - return false; -} - static std::string gen_item_id() { static std::atomic ctr{0}; char buf[32]; @@ -412,7 +377,7 @@ std::vector SseEmitter::emit_token(const std::string & raw_piece) { size_t think_close_idx = window_.find(THINK_CLOSE); size_t tool_idx = std::string::npos; bool tool_hit = has_request_tools(tools_) && - find_tool_start(window_, tool_idx); + find_tool_syntax_start(window_, tools_, tool_idx); struct Hit { size_t pos; int type; }; // type: 0=think, 1=think_close, 2=tool-ish std::vector hits; @@ -457,8 +422,10 @@ std::vector SseEmitter::emit_token(const std::string & raw_piece) { } // No tags found — emit safe prefix - if (window_.size() > std::max(BASE_HOLDBACK, stop_holdback_)) { - size_t cut = utf8_safe_len(window_, window_.size() - std::max(BASE_HOLDBACK, stop_holdback_)); + const size_t holdback = std::max(tool_syntax_holdback(tools_), + stop_holdback_); + if (window_.size() > holdback) { + size_t cut = utf8_safe_len(window_, window_.size() - holdback); // When tools are declared, a trailing identifier run may be a // Laguna tool name whose has not streamed in yet (the // wrapper is a stripped special token). Hold it back diff --git a/server/src/server/sse_emitter.h b/server/src/server/sse_emitter.h index 8dad2934f..e33b850a5 100644 --- a/server/src/server/sse_emitter.h +++ b/server/src/server/sse_emitter.h @@ -177,7 +177,9 @@ class SseEmitter { // Responses API IDs std::string msg_item_id_; - static constexpr size_t BASE_HOLDBACK = 12; // max(len(""), len(""), len("")) + // Longest tool/reasoning opener minus one, retained so an opener split + // across streamed tokens is still recognized on the next token. + static constexpr size_t BASE_HOLDBACK = 15; // len("V +// 10. TOOLV... +// 11. V... // // Pattern 5 runs *before* pattern 6 so that args like // call:outer{"name": "inner", "arguments": {}} @@ -50,6 +53,96 @@ static std::string generate_call_id() { return id; } +static const char TOOL_OPEN[] = ""; +static const char FUNCTION_OPEN[] = " 64) return false; + auto is_alpha_or_underscore = [](char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; + }; + if (!is_alpha_or_underscore(name.front())) return false; + for (const char c : name) { + if (!is_alpha_or_underscore(c) && !(c >= '0' && c <= '9') && + c != '.' && c != '-') { + return false; + } + } + return true; +} + +static std::string declared_tool_name(const json & tool) { + const auto & fn = tool.is_object() && tool.contains("function") + ? tool["function"] : tool; + const std::string name = fn.is_object() + ? fn.value("name", "") : std::string(); + return valid_tool_name(name) ? name : std::string(); +} + +static bool declared_tool_open_at(const std::string & text, size_t pos, + const json & tools) { + if (!tools.is_array()) return false; + for (const auto & tool : tools) { + const std::string name = declared_tool_name(tool); + if (name.empty()) continue; + const std::string opener = "<" + name + ">"; + if (text.compare(pos, opener.size(), opener) == 0) return true; + } + return false; +} + +bool find_tool_syntax_start(const std::string & text, const json & tools, + size_t & pos) { + size_t idx = text.find('<'); + while (idx != std::string::npos) { + if (text.compare(idx, sizeof(TOOL_OPEN) - 1, TOOL_OPEN) == 0 || + text.compare(idx, sizeof(FUNCTION_OPEN) - 1, FUNCTION_OPEN) == 0 || + text.compare(idx, sizeof(FUNCTION_SPACE_OPEN) - 1, + FUNCTION_SPACE_OPEN) == 0 || + text.compare(idx, sizeof(FUNCNAME_OPEN) - 1, FUNCNAME_OPEN) == 0 || + text.compare(idx, sizeof(TOOL_CODE_OPEN) - 1, TOOL_CODE_OPEN) == 0 || + text.compare(idx, sizeof(ATTRIBUTE_PARAMETER_OPEN) - 1, + ATTRIBUTE_PARAMETER_OPEN) == 0 || + declared_tool_open_at(text, idx, tools)) { + pos = idx; + return true; + } + if (text.compare(idx, sizeof(ARG_KEY_OPEN) - 1, ARG_KEY_OPEN) == 0) { + size_t start = idx; + auto is_ident = [](char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_' || c == '-'; + }; + while (start > 0 && is_ident(text[start - 1])) start--; + if (start < idx) { + pos = start; + return true; + } + } + idx = text.find('<', idx + 1); + } + return false; +} + +size_t tool_syntax_holdback(const json & tools) { + // Longest fixed opener is `` of ``. + holdback = std::max(holdback, name.size() + 1); + } + } + return holdback; +} + // Check if a function name is in the allowed tools list. static bool tool_allowed(const json & tools, const std::string & name) { if (tools.is_null() || !tools.is_array() || tools.empty()) return true; @@ -156,6 +249,29 @@ static size_t include_preceding_tool_call_open(const std::string & text, size_t return wrapper; } +static size_t include_following_tool_call_close(const std::string & text, + size_t pos, + const std::string & fn_name = {}) { + size_t close_pos = pos; + while (close_pos < text.size()) { + const char c = text[close_pos]; + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') break; + close_pos++; + } + static constexpr char TOOL_CALL_CLOSE[] = ""; + if (text.compare(close_pos, sizeof(TOOL_CALL_CLOSE) - 1, + TOOL_CALL_CLOSE) == 0) { + return close_pos + sizeof(TOOL_CALL_CLOSE) - 1; + } + if (!fn_name.empty()) { + const std::string tool_close = ""; + if (text.compare(close_pos, tool_close.size(), tool_close) == 0) { + return close_pos + tool_close.size(); + } + } + return pos; +} + // ─── Pattern regexes ──────────────────────────────────────────────────── // We use std::regex for portability. Compiled once (function-local static). @@ -190,6 +306,36 @@ static const std::regex & re_bare_tool_name_xml() { return r; } +static const std::regex & re_attribute_tool_xml() { + static std::regex r( + R"re(([\s\S]*?))re"); + return r; +} + +static const std::regex & re_attribute_parameter_xml() { + static std::regex r( + R"re(([\s\S]*?))re"); + return r; +} + +static const std::regex & re_funcname_tool_xml() { + static std::regex r( + R"(\s*([A-Za-z_][\w.\-]*)\s*([\s\S]*?))"); + return r; +} + +static const std::regex & re_space_function_xml() { + static std::regex r( + R"(([\s\S]*?)(?:\s*)?)"); + return r; +} + +static const std::regex & re_complete_parameter_xml() { + static std::regex r( + R"(([\s\S]*?))"); + return r; +} + static const std::regex & re_tool_code() { static std::regex r(R"(([\s\S]*?))"); return r; @@ -363,6 +509,35 @@ static json parse_xml_params(const std::string & region, const std::string & fn_ return args; } +// Parse only a body composed entirely of complete ... +// elements. Compatibility patterns for malformed function openers use this +// stricter path so partial model output never becomes guessed arguments. +static bool parse_complete_parameter_body(const std::string & body, + const std::string & fn_name, + const json & tools, + json & args) { + const json props = find_tool_properties(tools, fn_name); + args = json::object(); + bool found_param = false; + size_t cursor = 0; + + auto begin = std::sregex_iterator(body.begin(), body.end(), + re_complete_parameter_xml()); + auto end = std::sregex_iterator(); + for (auto it = begin; it != end; ++it) { + const size_t pos = it->position(); + if (!trim_ws(body.substr(cursor, pos - cursor)).empty()) return false; + + const std::string key = (*it)[1].str(); + if (args.contains(key)) return false; + args[key] = convert_param_value(trim_ws((*it)[2].str()), key, props); + found_param = true; + cursor = pos + it->length(); + } + + return found_param && trim_ws(body.substr(cursor)).empty(); +} + // ─── JSON tool call parser ────────────────────────────────────────────── // Parse {"name": ..., "arguments": ...} or {"function": {"name": ..., "arguments": ...}} @@ -600,6 +775,7 @@ static bool parse_function_sig_args(const std::string & arg_text, json & out_arg ToolParseResult parse_tool_calls(const std::string & text, const json & tools) { ToolParseResult result; std::vector removals; + std::vector> positioned_calls; auto add_call = [&](const std::string & fn_name, const json & args, size_t start, size_t end) { @@ -608,7 +784,7 @@ ToolParseResult parse_tool_calls(const std::string & text, const json & tools) { tc.id = generate_call_id(); tc.name = fn_name; tc.arguments = args.dump(); - result.tool_calls.push_back(std::move(tc)); + positioned_calls.emplace_back(start, std::move(tc)); removals.push_back({start, end}); }; @@ -776,8 +952,112 @@ ToolParseResult parse_tool_calls(const std::string & text, const json & tools) { std::string params = (*it)[2].str(); if (!tool_allowed(tools, fn_name)) continue; if (params.find("length()); + add_call(fn_name, parse_xml_params(params, fn_name, tools), pos, + include_following_tool_call_close(text, + pos + it->length(), + fn_name)); + } + } + + // Pattern 3c: VALUE... + // Some Qwen outputs confuse the tool wrapper with an attribute-style + // parameter tag. Accept this only when the outer name is a requested + // tool and the entire body consists of fully closed parameter elements. + // This strict body check prevents partial or guessed arguments. + if (tools.is_array() && !tools.empty()) { + auto begin = std::sregex_iterator(text.begin(), text.end(), re_attribute_tool_xml()); + auto end = std::sregex_iterator(); + for (auto it = begin; it != end; ++it) { + const size_t pos = it->position(); + if (overlaps(removals, pos)) continue; + + const std::string fn_name = (*it)[1].str(); + if (!tool_allowed(tools, fn_name)) continue; + + const std::string body = (*it)[2].str(); + const json props = find_tool_properties(tools, fn_name); + json args = json::object(); + bool valid = true; + bool found_param = false; + size_t cursor = 0; + + auto pbegin = + std::sregex_iterator(body.begin(), body.end(), re_attribute_parameter_xml()); + auto pend = std::sregex_iterator(); + for (auto pit = pbegin; pit != pend; ++pit) { + const size_t ppos = pit->position(); + if (!trim_ws(body.substr(cursor, ppos - cursor)).empty()) { + valid = false; + break; + } + + const std::string key = (*pit)[1].str(); + if (args.contains(key)) { + valid = false; + break; + } + args[key] = + convert_param_value(trim_ws((*pit)[2].str()), key, props); + found_param = true; + cursor = ppos + pit->length(); + } + + if (!trim_ws(body.substr(cursor)).empty()) valid = false; + if (!valid || !found_param) continue; + + add_call(fn_name, args, include_preceding_tool_call_open(text, pos), + include_following_tool_call_close(text, + pos + it->length())); + } + } + + // Pattern 3d: TOOLVALUE.... + // Qwen occasionally substitutes the literal `funcname` tag for + // ``. Accept it only for a requested tool and only when + // the remaining body is composed entirely of complete parameter blocks. + if (tools.is_array() && !tools.empty()) { + auto begin = std::sregex_iterator(text.begin(), text.end(), re_funcname_tool_xml()); + auto end = std::sregex_iterator(); + for (auto it = begin; it != end; ++it) { + const size_t pos = it->position(); + if (overlaps(removals, pos)) continue; + + const std::string fn_name = (*it)[1].str(); + if (!tool_allowed(tools, fn_name)) continue; + + json args; + if (!parse_complete_parameter_body((*it)[2].str(), fn_name, + tools, args)) { + continue; + } + + add_call(fn_name, args, include_preceding_tool_call_open(text, pos), + include_following_tool_call_close(text, + pos + it->length())); + } + } + + // Pattern 3e: VALUE.... + // This is a malformed Qwen variant of . Accept it only + // for a requested tool and only when every argument is fully delimited. + if (tools.is_array() && !tools.empty()) { + auto begin = std::sregex_iterator(text.begin(), text.end(), + re_space_function_xml()); + auto end = std::sregex_iterator(); + for (auto it = begin; it != end; ++it) { + const size_t pos = it->position(); + if (overlaps(removals, pos)) continue; + + const std::string fn_name = (*it)[1].str(); + if (!tool_allowed(tools, fn_name)) continue; + + json args; + if (!parse_complete_parameter_body((*it)[2].str(), fn_name, + tools, args)) { + continue; + } + add_call(fn_name, args, include_preceding_tool_call_open(text, pos), + pos + it->length()); } } @@ -897,6 +1177,18 @@ ToolParseResult parse_tool_calls(const std::string & text, const json & tools) { } } + // Detection runs by syntax family, so restore the calls' source order + // before exposing them to clients. Dependent calls must execute in the + // same order in which the model emitted them. + std::stable_sort(positioned_calls.begin(), positioned_calls.end(), + [](const auto & a, const auto & b) { + return a.first < b.first; + }); + result.tool_calls.reserve(positioned_calls.size()); + for (auto & entry : positioned_calls) { + result.tool_calls.push_back(std::move(entry.second)); + } + // Build cleaned text by removing all matched spans if (removals.empty()) { result.cleaned_text = text; diff --git a/server/src/server/tool_parser.h b/server/src/server/tool_parser.h index 55d48712a..0efa2724e 100644 --- a/server/src/server/tool_parser.h +++ b/server/src/server/tool_parser.h @@ -1,17 +1,22 @@ // Tool call parser — extracts structured tool calls from generated text. // -// Supports 7 detection patterns: +// Supports 11 detection patterns: // 1. ... (Qwen XML) // 2. ... (bare function XML) // 3. (function signature) -// 4. {...JSON...} (tool_code wrapper) -// 5. call:?{relaxed-JSON-args} (gemma plain-text) -// 6. Bare JSON objects {"name":..., "arguments":...} (raw JSON) -// 7. Whole-response JSON args for one declared tool {"arg":...} +// 4. v (bare tool tag) +// 5. ... (attribute XML) +// 6. namev (malformed XML) +// 7. v (malformed XML) +// 8. {...JSON...} (tool_code wrapper) +// 9. call:?{relaxed-JSON-args} (gemma plain-text) +// 10. Bare JSON objects {"name":..., "arguments":...} (raw JSON) +// 11. Whole-response JSON args for one declared tool {"arg":...} #pragma once #include +#include #include #include #include @@ -31,6 +36,16 @@ struct ToolParseResult { std::vector tool_calls; }; +// Find the first supported tool-call opener in streamed text. In addition to +// the built-in compatibility forms, this recognizes `` so the +// streaming emitter and final parser agree on bare tool-name syntax. +bool find_tool_syntax_start(const std::string & text, const json & tools, + size_t & pos); + +// Number of trailing bytes the streaming emitter must retain to recognize an +// opener split across token boundaries. +size_t tool_syntax_holdback(const json & tools); + // Parse tool calls from generated text. `tools` is the tool definitions // (used for type coercion and allow-list filtering). May be null/empty. ToolParseResult parse_tool_calls(const std::string & text, diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index 7a5455b44..8ee4250f2 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -147,6 +147,55 @@ static json shell_tools() { }); } +static json bash_tools() { + json tools = shell_tools(); + tools[0]["name"] = "bash"; + return tools; +} + +static json read_tools() { + return json::array({ + {{"type", "function"}, + {"function", { + {"name", "read"}, + {"parameters", { + {"type", "object"}, + {"properties", { + {"path", {{"type", "string"}}}, + {"offset", {{"type", "integer"}}}, + {"limit", {{"type", "integer"}}} + }}, + {"required", json::array({"path"})} + }} + }}} + }); +} + +static json read_and_bash_tools() { + json tools = read_tools(); + tools.push_back(bash_tools()[0]); + return tools; +} + +static json edit_tools() { + return json::array({ + {{"type", "function"}, + {"function", { + {"name", "edit"}, + {"parameters", { + {"type", "object"}, + {"properties", { + {"edits", { + {"type", "array"}, + {"items", {{"type", "object"}}} + }} + }}, + {"required", json::array({"edits"})} + }} + }}} + }); +} + static json optional_shell_tools() { json tools = shell_tools(); tools[0]["input_schema"].erase("required"); @@ -369,6 +418,210 @@ static void test_parse_bare_tool_name_xml_with_function_close() { std::string::npos); } +static void test_parse_repeated_bare_edit_calls_with_trailing_close() { + const std::string text = + "Applying both updates.\n\n" + "\n" + "\n" + "[{\"path\":\"/workspace/first.conf\",\"oldText\":\"auto\"," + "\"newText\":\"enabled\"}]\n" + "\n" + "\n\n" + "\n" + "\n" + "[{\"path\":\"/workspace/second.conf\",\"oldText\":\"auto\"," + "\"newText\":\"enabled\"}]\n" + "\n" + "\n\n" + ""; + + auto result = parse_tool_calls(text, edit_tools()); + TEST_ASSERT(result.tool_calls.size() == 2); + if (result.tool_calls.size() == 2) { + TEST_ASSERT(result.tool_calls[0].name == "edit"); + TEST_ASSERT(result.tool_calls[1].name == "edit"); + const auto first = json::parse(result.tool_calls[0].arguments); + const auto second = json::parse(result.tool_calls[1].arguments); + TEST_ASSERT(first["edits"][0]["path"] == "/workspace/first.conf"); + TEST_ASSERT(second["edits"][0]["path"] == "/workspace/second.conf"); + } + TEST_ASSERT(result.cleaned_text == "Applying both updates."); +} + +static void test_tool_syntax_scanner_declared_name_guards() { + size_t pos = std::string::npos; + TEST_ASSERT(find_tool_syntax_start("prefix", edit_tools(), pos)); + TEST_ASSERT(pos == 6); + + pos = std::string::npos; + TEST_ASSERT(!find_tool_syntax_start("prefix", edit_tools(), pos)); + + json invalid = edit_tools(); + invalid[0]["function"]["name"] = std::string(65, 'x'); + TEST_ASSERT(tool_syntax_holdback(invalid) == 15); + pos = std::string::npos; + TEST_ASSERT(!find_tool_syntax_start("<" + std::string(65, 'x') + ">", + invalid, pos)); +} + +static void test_parse_undeclared_file_tag_stays_content() { + const std::string text = + "Now I understand how to add a custom model. I need to edit " + "~/.pi/agent/models.json to add a vLLM provider with the " + "laguna-s-2.1 model. Let me check if the file exists first.\n\n" + "\n" + "\n" + "~/.pi/agent/models.json\n" + "\n" + ""; + + auto result = parse_tool_calls(text, read_tools()); + TEST_ASSERT(result.tool_calls.empty()); + TEST_ASSERT(result.cleaned_text == text); +} + +static void test_parse_attribute_style_tool_xml() { + std::string text = + "The branch already exists. Let me check the current state:\n\n" + "" + "cd /workspace/project && " + "git status -sb && git branch\n --show-current\n" + "\n"; + auto result = parse_tool_calls(text, bash_tools()); + TEST_ASSERT(result.tool_calls.size() == 1); + if (!result.tool_calls.empty()) { + TEST_ASSERT(result.tool_calls[0].name == "bash"); + auto args = json::parse(result.tool_calls[0].arguments); + TEST_ASSERT(args["command"] == + "cd /workspace/project && " + "git status -sb && git branch\n --show-current"); + } + TEST_ASSERT(result.cleaned_text == + "The branch already exists. Let me check the current state:"); + + const std::string wrapped = + "Checking now.\n\n\n" + "" + "git status\n\n"; + auto wrapped_result = parse_tool_calls(wrapped, bash_tools()); + TEST_ASSERT(wrapped_result.tool_calls.size() == 1); + TEST_ASSERT(wrapped_result.cleaned_text == "Checking now."); +} + +static void test_parse_mixed_tool_variants_preserve_source_order() { + const std::string text = + "\n" + "/tmp/first.md\n" + "\n" + "" + "cat /tmp/first.md"; + auto result = parse_tool_calls(text, read_and_bash_tools()); + TEST_ASSERT(result.tool_calls.size() == 2); + if (result.tool_calls.size() == 2) { + TEST_ASSERT(result.tool_calls[0].name == "read"); + TEST_ASSERT(result.tool_calls[1].name == "bash"); + } + TEST_ASSERT(result.cleaned_text.empty()); +} + +static void test_parse_attribute_style_tool_xml_rejects_malformed_body() { + const std::string malformed = + "git status\n" + ""; + auto malformed_result = parse_tool_calls(malformed, bash_tools()); + TEST_ASSERT(malformed_result.tool_calls.empty()); + TEST_ASSERT(malformed_result.cleaned_text == malformed); + + const std::string unknown = + "" + "git status"; + auto unknown_result = parse_tool_calls(unknown, bash_tools()); + TEST_ASSERT(unknown_result.tool_calls.empty()); + TEST_ASSERT(unknown_result.cleaned_text == unknown); +} + +static void test_parse_funcname_tool_xml() { + const std::string text = + "\n" + "read\n" + "\n50\n\n" + "\n1\n\n" + "\n" + "/tmp/tool-input.md\n" + "\n" + "\n" + "\n"; + auto result = parse_tool_calls(text, read_tools()); + TEST_ASSERT(result.tool_calls.size() == 1); + if (!result.tool_calls.empty()) { + TEST_ASSERT(result.tool_calls[0].name == "read"); + auto args = json::parse(result.tool_calls[0].arguments); + TEST_ASSERT(args["limit"] == 50); + TEST_ASSERT(args["offset"] == 1); + TEST_ASSERT(args["path"] == "/tmp/tool-input.md"); + } + TEST_ASSERT(result.cleaned_text.empty()); +} + +static void test_parse_funcname_tool_xml_rejects_malformed_or_unknown() { + const std::string malformed = + "read\n" + "/tmp/task.md\n" + ""; + auto malformed_result = parse_tool_calls(malformed, read_tools()); + TEST_ASSERT(malformed_result.tool_calls.empty()); + TEST_ASSERT(malformed_result.cleaned_text == malformed); + + const std::string unknown = + "write\n" + "/tmp/task.md\n" + ""; + auto unknown_result = parse_tool_calls(unknown, read_tools()); + TEST_ASSERT(unknown_result.tool_calls.empty()); + TEST_ASSERT(unknown_result.cleaned_text == unknown); +} + +static void test_parse_space_function_tool_xml() { + const std::string text = + "Let me read the file and compute its SHA-256 hash.\n\n" + "\n" + "\n" + "/tmp/tool-input.md\n" + "\n" + "\n\n" + "\n" + "\n" + "sha256sum /tmp/tool-input.md\n" + "\n" + ""; + auto result = parse_tool_calls(text, read_and_bash_tools()); + TEST_ASSERT(result.tool_calls.size() == 2); + if (result.tool_calls.size() == 2) { + TEST_ASSERT(result.tool_calls[0].name == "read"); + TEST_ASSERT(json::parse(result.tool_calls[0].arguments)["path"] == + "/tmp/tool-input.md"); + TEST_ASSERT(result.tool_calls[1].name == "bash"); + TEST_ASSERT(json::parse(result.tool_calls[1].arguments)["command"] == + "sha256sum /tmp/tool-input.md"); + } + TEST_ASSERT(result.cleaned_text == + "Let me read the file and compute its SHA-256 hash."); +} + +static void test_parse_space_function_tool_xml_rejects_malformed_or_unknown() { + const std::string malformed = + "\n/tmp/task.md\n"; + auto malformed_result = parse_tool_calls(malformed, read_tools()); + TEST_ASSERT(malformed_result.tool_calls.empty()); + TEST_ASSERT(malformed_result.cleaned_text == malformed); + + const std::string unknown = + "\n/tmp/task.md\n"; + auto unknown_result = parse_tool_calls(unknown, read_tools()); + TEST_ASSERT(unknown_result.tool_calls.empty()); + TEST_ASSERT(unknown_result.cleaned_text == unknown); +} + static void test_parse_json_tool_call() { std::string text = "{\"name\": \"search\", \"arguments\": {\"query\": \"hello world\"}}"; @@ -1048,6 +1301,106 @@ static void test_emitter_bare_function_tool_buffer_detection() { TEST_ASSERT(em.accumulated_text().find("") == std::string::npos); } +static void test_emitter_attribute_style_tool_buffer_detection() { + auto em = make_emitter(ApiFormat::OPENAI_CHAT, bash_tools()); + em.emit_start(); + em.emit_token("The branch already exists. Let me check the current state:\n\n" + "" + "git status -sb && git branch\n"); + em.emit_token(" --show-current\n"); + auto finish = em.emit_finish(20); + const std::string wire = concat(finish); + + TEST_ASSERT(em.tool_calls().size() == 1); + if (!em.tool_calls().empty()) { + TEST_ASSERT(em.tool_calls()[0].name == "bash"); + auto args = json::parse(em.tool_calls()[0].arguments); + TEST_ASSERT(args["command"] == + "git status -sb && git branch\n --show-current"); + } + TEST_ASSERT(em.accumulated_text() == + "The branch already exists. Let me check the current state:\n\n"); + TEST_ASSERT(em.accumulated_text().find("read\n\n50\n\n" + "\n1\n\n"); + em.emit_token("\n" + "/tmp/tool-input.md\n" + "\n\n"); + auto finish = em.emit_finish(88); + const std::string wire = concat(finish); + + TEST_ASSERT(em.tool_calls().size() == 1); + if (!em.tool_calls().empty()) { + TEST_ASSERT(em.tool_calls()[0].name == "read"); + auto args = json::parse(em.tool_calls()[0].arguments); + TEST_ASSERT(args["limit"] == 50); + TEST_ASSERT(args["offset"] == 1); + TEST_ASSERT(args["path"] == "/tmp/tool-input.md"); + } + TEST_ASSERT(em.accumulated_text().find("") == std::string::npos); + TEST_ASSERT(em.accumulated_text() == "\n\n"); + TEST_ASSERT(wire.find("\"finish_reason\":\"tool_calls\"") != + std::string::npos); +} + +static void test_emitter_space_function_tool_buffer_detection() { + auto em = make_emitter(ApiFormat::OPENAI_CHAT, read_tools()); + em.emit_start(); + em.emit_token("Let me read it.\n\n\n\n"); + em.emit_token("/tmp/tool-input.md\n" + "\n"); + const std::string wire = concat(em.emit_finish(42)); + + TEST_ASSERT(em.tool_calls().size() == 1); + if (!em.tool_calls().empty()) { + TEST_ASSERT(em.tool_calls()[0].name == "read"); + TEST_ASSERT(json::parse(em.tool_calls()[0].arguments)["path"] == + "/tmp/tool-input.md"); + } + TEST_ASSERT(em.accumulated_text() == "Let me read it.\n\n"); + TEST_ASSERT(em.accumulated_text().find("") == + std::string::npos); + TEST_ASSERT(wire.find("\"finish_reason\":\"tool_calls\"") != + std::string::npos); +} + +static void test_emitter_repeated_bare_edit_calls() { + auto em = make_emitter(ApiFormat::OPENAI_CHAT, edit_tools()); + em.emit_start(); + em.emit_token("Applying both updates.\n\n\n\n" + "[{\"path\":\"/workspace/first.conf\"," + "\"oldText\":\"auto\",\"newText\":\"enabled\"}]\n" + "\n\n\n\n"); + em.emit_token("\n" + "[{\"path\":\"/workspace/second.conf\"," + "\"oldText\":\"auto\",\"newText\":\"enabled\"}]\n" + "\n\n\n"); + const std::string wire = concat(em.emit_finish(96)); + + TEST_ASSERT(em.tool_calls().size() == 2); + if (em.tool_calls().size() == 2) { + const auto first = json::parse(em.tool_calls()[0].arguments); + const auto second = json::parse(em.tool_calls()[1].arguments); + TEST_ASSERT(first["edits"][0]["path"] == "/workspace/first.conf"); + TEST_ASSERT(second["edits"][0]["path"] == "/workspace/second.conf"); + } + TEST_ASSERT(em.accumulated_text() == "Applying both updates.\n\n"); + TEST_ASSERT(em.accumulated_text().find("") == std::string::npos); + TEST_ASSERT(wire.find("\"finish_reason\":\"tool_calls\"") != + std::string::npos); +} + static void test_emitter_does_not_leak_malformed_tool_xml() { auto em = make_emitter(ApiFormat::OPENAI_CHAT, weather_tools()); em.emit_start(); @@ -1098,6 +1451,24 @@ static void test_emitter_no_tools_keeps_tool_like_text() { TEST_ASSERT(em.accumulated_text().find("") != std::string::npos); } +static void test_emitter_undeclared_file_tag_stays_content() { + const std::string text = + "Let me check if the file exists first.\n\n" + "\n" + "\n" + "~/.pi/agent/models.json\n" + "\n" + ""; + + auto em = make_emitter(ApiFormat::OPENAI_CHAT, read_tools()); + em.emit_start(); + em.emit_token(text); + em.emit_finish(20); + + TEST_ASSERT(em.tool_calls().empty()); + TEST_ASSERT(em.accumulated_text() == text); +} + static void test_emitter_anthropic_structure() { // Verify Anthropic format emits proper event sequence. auto em = make_emitter(ApiFormat::ANTHROPIC); @@ -4460,6 +4831,16 @@ int main() { RUN_TEST(test_parse_tool_call_xml); RUN_TEST(test_parse_bare_function_xml); RUN_TEST(test_parse_bare_tool_name_xml_with_function_close); + RUN_TEST(test_parse_repeated_bare_edit_calls_with_trailing_close); + RUN_TEST(test_tool_syntax_scanner_declared_name_guards); + RUN_TEST(test_parse_undeclared_file_tag_stays_content); + RUN_TEST(test_parse_attribute_style_tool_xml); + RUN_TEST(test_parse_mixed_tool_variants_preserve_source_order); + RUN_TEST(test_parse_attribute_style_tool_xml_rejects_malformed_body); + RUN_TEST(test_parse_funcname_tool_xml); + RUN_TEST(test_parse_funcname_tool_xml_rejects_malformed_or_unknown); + RUN_TEST(test_parse_space_function_tool_xml); + RUN_TEST(test_parse_space_function_tool_xml_rejects_malformed_or_unknown); RUN_TEST(test_parse_json_tool_call); RUN_TEST(test_parse_single_tool_bare_json_args); RUN_TEST(test_parse_single_tool_bare_json_args_allows_empty_optional_object); @@ -4509,9 +4890,14 @@ int main() { RUN_TEST(test_emitter_single_tool_bare_json_args); RUN_TEST(test_emitter_bare_json_args_do_not_trigger_after_content); RUN_TEST(test_emitter_bare_function_tool_buffer_detection); + RUN_TEST(test_emitter_attribute_style_tool_buffer_detection); + RUN_TEST(test_emitter_funcname_tool_buffer_detection); + RUN_TEST(test_emitter_space_function_tool_buffer_detection); + RUN_TEST(test_emitter_repeated_bare_edit_calls); RUN_TEST(test_emitter_does_not_leak_malformed_tool_xml); RUN_TEST(test_emitter_parses_tool_call_missing_outer_close); RUN_TEST(test_emitter_no_tools_keeps_tool_like_text); + RUN_TEST(test_emitter_undeclared_file_tag_stays_content); RUN_TEST(test_emitter_anthropic_structure); RUN_TEST(test_emitter_responses_structure); RUN_TEST(test_emitter_responses_bare_function_tool_call);