From 5a196a78576f3450f8046b2a2a3bd4ad54fac92b Mon Sep 17 00:00:00 2001 From: Mirko Perrone Date: Sun, 12 Apr 2026 23:55:49 +0200 Subject: [PATCH 1/3] feat(proxy): improve payload handling and stabilize ChatJimmy integration - Added message cleaning to remove empty/invalid contents before sending upstream - Introduced safe system prompt (truncated to 8k, removed tool injection) to prevent empty responses - Simplified jimmy_payload structure for better compatibility with chatjimmy.ai - Added fallback default message when message list is empty - Improved logging with clearer request/response tracing - Preserved tool parsing logic while preventing oversized system prompts (>28k) - Fixed issues causing upstream failures and empty responses --- proxy.py | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/proxy.py b/proxy.py index e9f096c..0189059 100644 --- a/proxy.py +++ b/proxy.py @@ -481,16 +481,44 @@ def do_POST(self): ) full_system_prompt = full_system_prompt[:MAX_SYSTEM_PROMPT] + # jimmy_payload = { + # "messages": chat_messages, + # "chatOptions": { + # "selectedModel": MODELS.get(model, model), + # "systemPrompt": full_system_prompt, + # "topK": 8, + # }, + # "attachment": None, + # } + # 🔧 CLEAN messages (rimuove vuoti e roba strana) + clean_messages = [] + for m in chat_messages: + content = m.get("content", "") + if not content or not str(content).strip(): + continue + + clean_messages.append({ + "role": m.get("role", "user"), + "content": str(content) + }) + + # 🔧 system prompt safe (NO tools, NO roba lunga) + safe_system_prompt = (system_prompt or "").strip() + + if len(safe_system_prompt) > 8000: + safe_system_prompt = safe_system_prompt[:8000] + + # 🔥 PAYLOAD FINALE FIXATO jimmy_payload = { - "messages": chat_messages, + "messages": clean_messages if clean_messages else [ + {"role": "user", "content": "Hello"} + ], "chatOptions": { - "selectedModel": MODELS.get(model, model), - "systemPrompt": full_system_prompt, - "topK": 8, - }, - "attachment": None, + "selectedModel": MODELS.get(model, "llama3.1-8B"), + "systemPrompt": safe_system_prompt, + "topK": 8 + } } - # File: translated payload logfile("--- TRANSLATED PAYLOAD ---") logfile(f"{json.dumps(jimmy_payload, indent=2)}") From 733cbe0925e737f241df721d87aab8859721d11b Mon Sep 17 00:00:00 2001 From: Mirko Perrone Date: Tue, 21 Apr 2026 19:12:38 +0200 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=90=9B=20fix(proxy):=20migliorare=20g?= =?UTF-8?q?estione=20prompt=20sistema=20e=20validazione=20risposte?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Cambiato default tool_choice da "auto" a "required" per garantire che gli strumenti vengano chiamati quando necessario - Aggiunti limiti di lunghezza separati per base system prompt (20K) e total system prompt (80K) per evitare risposte vuote da upstream - Migliorata pulizia dei messaggi: rimossi contenuti vuoti, mantenuti tutti i ruoli validi - Aggiunta fallback per messaggi vuoti per garantire almeno un messaggio utente nel payload - Aggiunti warning log per risposte vuote o troppo brevi dall'upstream con dettagli su lunghezza prompt e strumenti [body] --- proxy.py | 79 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/proxy.py b/proxy.py index 0189059..f0d1e72 100644 --- a/proxy.py +++ b/proxy.py @@ -394,7 +394,7 @@ def do_POST(self): for t in openai_req.get("tools", []) if t.get("function", {}).get("name", "").lower() not in FILTERED_TOOLS ] - tool_choice = openai_req.get("tool_choice", "auto") + tool_choice = openai_req.get("tool_choice", "required") last_content = extract_text_content( messages[-1].get("content", "") if messages else "" @@ -468,57 +468,54 @@ def do_POST(self): else: chat_messages.append({"role": role, "content": content}) - # Append tool definitions to the system prompt - full_system_prompt = system_prompt.strip() - if tools: - full_system_prompt += format_tools_for_prompt(tools, tool_choice) + # ----- Build final system prompt (base + tools) ----- + # chatjimmy silently returns empty responses above ~8-10K chars total. + # Budget: 2000 for base system, rest for tools. + MAX_BASE_SYSTEM = 20000 + MAX_TOTAL_SYSTEM = 80000 - # ChatJimmy returns empty responses when system prompt exceeds ~30K chars - MAX_SYSTEM_PROMPT = 28000 - if len(full_system_prompt) > MAX_SYSTEM_PROMPT: + base_system_prompt = system_prompt.strip() + if len(base_system_prompt) > MAX_BASE_SYSTEM: logfile( - f"WARNING: system prompt is {len(full_system_prompt)} chars, truncating to {MAX_SYSTEM_PROMPT}" + f"WARNING: base system prompt is {len(base_system_prompt)} chars, truncating to {MAX_BASE_SYSTEM}" ) - full_system_prompt = full_system_prompt[:MAX_SYSTEM_PROMPT] - - # jimmy_payload = { - # "messages": chat_messages, - # "chatOptions": { - # "selectedModel": MODELS.get(model, model), - # "systemPrompt": full_system_prompt, - # "topK": 8, - # }, - # "attachment": None, - # } - # 🔧 CLEAN messages (rimuove vuoti e roba strana) + base_system_prompt = base_system_prompt[:MAX_BASE_SYSTEM] + + # Always append tool definitions so the model knows how to call them + tools_section = format_tools_for_prompt(tools, tool_choice) if tools else "" + full_system_prompt = base_system_prompt + tools_section + + if len(full_system_prompt) > MAX_TOTAL_SYSTEM: + logfile( + f"WARNING: full system prompt is {len(full_system_prompt)} chars, truncating to {MAX_TOTAL_SYSTEM}. " + f"Consider reducing the number of tools (currently {len(tools)})." + ) + log(f"WARNING: system prompt too large ({len(full_system_prompt)} chars with {len(tools)} tools), truncating") + full_system_prompt = full_system_prompt[:MAX_TOTAL_SYSTEM] + + # Clean messages: drop empty content, keep all valid roles clean_messages = [] for m in chat_messages: - content = m.get("content", "") - if not content or not str(content).strip(): + msg_content = m.get("content", "") + if not msg_content or not str(msg_content).strip(): continue - clean_messages.append({ "role": m.get("role", "user"), - "content": str(content) + "content": str(msg_content), }) - # 🔧 system prompt safe (NO tools, NO roba lunga) - safe_system_prompt = (system_prompt or "").strip() - - if len(safe_system_prompt) > 8000: - safe_system_prompt = safe_system_prompt[:8000] + if not clean_messages: + clean_messages = [{"role": "user", "content": "Hello"}] - # 🔥 PAYLOAD FINALE FIXATO jimmy_payload = { - "messages": clean_messages if clean_messages else [ - {"role": "user", "content": "Hello"} - ], + "messages": clean_messages, "chatOptions": { "selectedModel": MODELS.get(model, "llama3.1-8B"), - "systemPrompt": safe_system_prompt, - "topK": 8 - } + "systemPrompt": full_system_prompt, + "topK": 8, + }, } + # File: translated payload logfile("--- TRANSLATED PAYLOAD ---") logfile(f"{json.dumps(jimmy_payload, indent=2)}") @@ -554,6 +551,12 @@ def do_POST(self): logfile("--- RAW UPSTREAM RESPONSE ---") logfile(raw_response) + # Warn on empty or suspiciously short responses + if not raw_response.strip(): + log(f"WARNING: upstream returned empty response (system_prompt={len(full_system_prompt)} chars, tools={len(tools)})") + elif len(raw_response.strip()) < 10: + log(f"WARNING: upstream returned very short response: {repr(raw_response)}") + # Strip stats, parse usage content = re.sub( r"<\|stats\|>.*?<\|/stats\|>", "", raw_response, flags=re.DOTALL @@ -754,4 +757,4 @@ def main(): if __name__ == "__main__": - main() + main() \ No newline at end of file From 305bc79d5526894e038b8f1a2315e124a1d93178 Mon Sep 17 00:00:00 2001 From: Mirko Perrone Date: Sun, 26 Apr 2026 19:56:28 +0200 Subject: [PATCH 3/3] fix(core): add fallback for raw tool calls and improve prompt handling The model sometimes forgets to wrap tool calls within tags, causing errors. This change adds a fallback that detects raw JSON and handles it properly. Additionally, the prompt management has been optimized to ensure tools are never truncated, improving tool call reliability. implement detection of bare JSON tool calls when tags are missing add cleanup logic to remove bare JSON from returned text redesign prompt handling to prioritize tool definitions tools are now always included and never truncated add detailed logging on prompt sizes --- proxy.py | 55 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/proxy.py b/proxy.py index f0d1e72..dd73fac 100644 --- a/proxy.py +++ b/proxy.py @@ -213,12 +213,26 @@ def _extract_call_objects(obj): def parse_tool_calls(content, tools=None): """ Parse … blocks from the model's text. + Also handles bare JSON tool calls without tags (fallback). Returns (text_without_tags, list_of_openai_tool_call_dicts). """ pattern = re.compile(r"\s*(.*?)\s*", re.DOTALL) matches = pattern.findall(content) + # Fallback: detect bare JSON like {"name": "...", "arguments": {...}} + # when the model forgets to wrap in tags + bare_match = None + if not matches: + bare_pattern = re.compile( + r'(\{[^{}]*"name"\s*:\s*"[^"]+"\s*,\s*"arguments"\s*:\s*\{.*?\}[^{}]*\})', + re.DOTALL, + ) + bare_matches = bare_pattern.findall(content) + if bare_matches: + matches = bare_matches + bare_match = True + if not matches: return content, [] @@ -263,7 +277,15 @@ def parse_tool_calls(content, tools=None): except (json.JSONDecodeError, KeyError, AttributeError): continue - text = pattern.sub("", content).strip() + if bare_match: + # Remove the matched bare JSON blobs from text + bare_pattern = re.compile( + r'(\{[^{}]*"name"\s*:\s*"[^"]+"\s*,\s*"arguments"\s*:\s*\{.*?\}[^{}]*\})', + re.DOTALL, + ) + text = bare_pattern.sub("", content).strip() + else: + text = pattern.sub("", content).strip() return text, tool_calls @@ -468,30 +490,29 @@ def do_POST(self): else: chat_messages.append({"role": role, "content": content}) - # ----- Build final system prompt (base + tools) ----- - # chatjimmy silently returns empty responses above ~8-10K chars total. - # Budget: 2000 for base system, rest for tools. - MAX_BASE_SYSTEM = 20000 + # ----- Build final system prompt (tools first, then base) ----- + # chatjimmy silently returns empty responses above ~8000 chars. + # Strategy: tools are ALWAYS included intact (they must not be truncated), + # and the base system prompt is trimmed to fit within the budget. MAX_TOTAL_SYSTEM = 80000 + tools_section = format_tools_for_prompt(tools, tool_choice) if tools else "" + tools_len = len(tools_section) + + base_budget = max(0, MAX_TOTAL_SYSTEM - tools_len) base_system_prompt = system_prompt.strip() - if len(base_system_prompt) > MAX_BASE_SYSTEM: + + if len(base_system_prompt) > base_budget: logfile( - f"WARNING: base system prompt is {len(base_system_prompt)} chars, truncating to {MAX_BASE_SYSTEM}" + f"WARNING: base system prompt truncated from {len(base_system_prompt)} to {base_budget} chars " + f"(tools use {tools_len} chars)" ) - base_system_prompt = base_system_prompt[:MAX_BASE_SYSTEM] + base_system_prompt = base_system_prompt[:base_budget] - # Always append tool definitions so the model knows how to call them - tools_section = format_tools_for_prompt(tools, tool_choice) if tools else "" + # Tools go LAST so they are never cut off by truncation full_system_prompt = base_system_prompt + tools_section - if len(full_system_prompt) > MAX_TOTAL_SYSTEM: - logfile( - f"WARNING: full system prompt is {len(full_system_prompt)} chars, truncating to {MAX_TOTAL_SYSTEM}. " - f"Consider reducing the number of tools (currently {len(tools)})." - ) - log(f"WARNING: system prompt too large ({len(full_system_prompt)} chars with {len(tools)} tools), truncating") - full_system_prompt = full_system_prompt[:MAX_TOTAL_SYSTEM] + log(f"system_prompt={len(full_system_prompt)} chars (base={len(base_system_prompt)}, tools={tools_len})") # Clean messages: drop empty content, keep all valid roles clean_messages = []