R1 distill qwen

functionary/openai_types.py

@@ -121,7 +121,7 @@ class ChatCompletionRequest(BaseModel):
     function_call: Optional[Union[str, Function]] = None
     tool_choice: Optional[Union[str, Tool]] = None
     temperature: Optional[float] = 0.6
-    top_p: Optional[float] = 1.0
+    top_p: Optional[float] = 0.95
     n: Optional[int] = 1
     max_tokens: Optional[int] = 4096
     stop: Optional[Union[str, List[str]]] = Field(default_factory=list)

functionary/prompt_template/__init__.py

@@ -8,8 +8,15 @@
 from functionary.prompt_template.llava_prompt_template import LlavaLlama
 from functionary.prompt_template.prompt_template_v1 import PromptTemplateV1
 from functionary.prompt_template.prompt_template_v2 import PromptTemplateV2
-from functionary.prompt_template.llama31_reasoning_prompt_template import Llama31ReasoningTemplate
-
+from functionary.prompt_template.llama31_reasoning_prompt_template import (
+    Llama31ReasoningTemplate,
+)
+from functionary.prompt_template.r1_original_template import R1Template
+from functionary.prompt_template.r1_distilled_qwen import R1DistilledQwen
+from functionary.prompt_template.r1_distilled_llama import R1DistilledLlama
+from functionary.prompt_template.qwen25_text_only_template import Qwen25TextOnlyPromptTemplate
+from functionary.prompt_template.gemma3_prompt_template import Gemma3Template
+from functionary.prompt_template.cogito_prompt_template import CogitoPromptTemplate
 
 def get_available_prompt_template_versions() -> List[PromptTemplate]:
     """This function will get all the available prompt templates in the module.
@@ -30,6 +37,10 @@ def get_available_prompt_template_versions() -> List[PromptTemplate]:
     # we don't use get_prompt_template or this will return the parent class
     all_templates_obj.append(LlavaLlama.get_prompt_template())
     all_templates_obj.append(Llama31ReasoningTemplate.get_prompt_template())
+    all_templates_obj.append(R1DistilledLlama.get_prompt_template())
+    all_templates_obj.append(R1DistilledQwen.get_prompt_template())
+    all_templates_obj.append(Gemma3Template.get_prompt_template())
+    all_templates_obj.append(CogitoPromptTemplate.get_prompt_template())
     return all_templates_obj
 
 

functionary/prompt_template/base_template.py

@@ -69,13 +69,12 @@ def initialize_grammar_sampling_gen_state(self, tool_choice: Optional[Any]) -> D
         """
         raise NotImplementedError
 
-    @abstractmethod
     def get_additional_tokens(self) -> List[str]:
         """return list of added tokens if using this template
         Returns:
             List[str]: list of tokens, each token is a string
         """
-        raise NotImplementedError
+        return []
 
     @abstractmethod
     def get_stop_tokens_for_generation(self) -> List[str]:

functionary/prompt_template/cogito_prompt_template.py

@@ -0,0 +1,17 @@
+from functionary.prompt_template.base_template import PromptTemplate
+from functionary.prompt_template.qwen25_text_only_template import (
+    Qwen25TextOnlyPromptTemplate,
+)
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+from functionary.prompt_template import prompt_utils
+from functionary.openai_types import Function, Tool
+
+
+class CogitoPromptTemplate(Qwen25TextOnlyPromptTemplate):
+    version = "cogito"
+
+    def get_chat_template_jinja(self) -> str:
+        path_prefix = "./functionary/prompt_template/jinja_templates/"
+        with open(f"{path_prefix}cogito.txt", "r") as f:
+            template = f.read()
+        return template

functionary/prompt_template/gemma3_prompt_template.py

@@ -0,0 +1,38 @@
+import json
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+
+from functionary.openai_types import Function, Tool
+from functionary.prompt_template import prompt_utils
+from functionary.prompt_template.qwen25_text_only_template import (
+    Qwen25TextOnlyPromptTemplate,
+)
+
+
+class Gemma3Template(Qwen25TextOnlyPromptTemplate):
+    version = "gemma3"
+
+    def get_additional_tokens(self) -> List[str]:
+        return []
+
+    def get_assistant_prefixes(self) -> List[str]:
+        return ["<start_of_turn>model\n"]
+
+    def get_stop_tokens_for_generation(self) -> List[str]:
+        return ["<end_of_turn>"]
+
+    def pre_process_messages_before_inference(self, messages: List[Dict]) -> List[Dict]:
+        """Order the tool results by the order of tool call ids
+
+        Args:
+            messages (List[Dict]): List of messages
+
+        Returns:
+            List[Dict]: List of messages
+        """
+        return prompt_utils.reorder_tool_messages_by_tool_call_ids(messages)
+
+    def get_chat_template_jinja(self) -> str:
+        path_prefix = "./functionary/prompt_template/jinja_templates/"
+        with open(f"{path_prefix}{self.version}.txt", "r") as f:
+            template = f.read()
+        return template

functionary/prompt_template/jinja_templates/cogito.txt

@@ -0,0 +1,80 @@
+{# version=cogito #}{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages and messages[0]['role'] == 'system' %}
+        {{- 'Enable deep thinking subroutine.\n\n' + messages[0]['content'] }}
+    {%- else %}
+        {{- 'Enable deep thinking subroutine.' }}
+    {%- endif %}
+    {%- set has_reasoning = tools | selectattr("type", "equalto", "reasoning") | list | length > 0 -%}
+    {%- if has_reasoning -%}
+        {%- set tools = tools | rejectattr("type", "equalto", "reasoning") | list -%}
+    {%- endif -%}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query." }}
+    {%- if has_reasoning %}
+        {{- " Please generate reasoning before deciding which tools to use." }}
+    {%- else %}
+        {{- "" }}
+    {%- endif %}
+    {{- "\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages and messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\nEnable deep thinking subroutine\n\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nEnable deep thinking subroutine.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message.role == "user" %}
+        {%- if message['content'] -%}
+            {%- if message['content'] is string -%}
+                {{ '<|im_start|>user\n' + message['content'] }}
+            {%- else -%}
+                {{ '<|im_start|>user\n' }}
+                {%- for content in message['content'] -%}
+                    {%- if content['type'] == 'text' -%}
+                        {{ content['text'] }}
+                    {%- else -%}
+                        {{ '<|vision_start|><|image_pad|><|vision_end|>' }}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+            {{ '<|im_end|>\n' }}
+        {%- endif -%}
+    {%- elif (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

functionary/prompt_template/jinja_templates/gemma3.txt

@@ -0,0 +1,64 @@
+{# version=gemma3 #}{{- bos_token -}}{%- if tools %}
+    {{- '<start_of_turn>system\nYou are capable of executing available function(s) if required.' }}
+    {%- set has_reasoning = tools | selectattr("type", "equalto", "reasoning") | list | length > 0 -%}
+    {%- if has_reasoning -%}
+        {%- set tools = tools | rejectattr("type", "equalto", "reasoning") | list -%}
+    {%- endif -%}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query." }}
+    {%- if has_reasoning %}
+        {{- " Please generate reasoning before deciding which tools to use." }}
+    {%- else %}
+        {{- "" }}
+    {%- endif %}
+    {{- "\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><end_of_turn>\n" }}
+{%- else %}
+    {%- if messages and messages[0]['role'] == 'system' %}
+        {{- '<start_of_turn>system\n' + messages[0]['content'] + '<end_of_turn>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system") -%}
+        {{- '<start_of_turn>' + message.role + '\n' + message.content + '<end_of_turn>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<start_of_turn>model' }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+
+        {%- if "tool_calls" in message and message["tool_calls"] is not none -%}
+            {%- set tool_calls = message["tool_calls"] -%}
+        {%-else-%}
+            {%- set tool_calls = [] -%}
+        {%- endif -%}
+
+        {%- for tool_call in tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<end_of_turn>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<start_of_turn>tool' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<end_of_turn>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<start_of_turn>model\n' }}
+{%- endif %}

functionary/prompt_template/jinja_templates/qwen2.5-text-only.txt

@@ -0,0 +1,80 @@
+{# version=qwen2.5-text-only #}{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages and messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are a helpful assistant.' }}
+    {%- endif %}
+    {%- set has_reasoning = tools | selectattr("type", "equalto", "reasoning") | list | length > 0 -%}
+    {%- if has_reasoning -%}
+        {%- set tools = tools | rejectattr("type", "equalto", "reasoning") | list -%}
+    {%- endif -%}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query." }}
+    {%- if has_reasoning %}
+        {{- " Please generate reasoning before deciding which tools to use." }}
+    {%- else %}
+        {{- "" }}
+    {%- endif %}
+    {{- "\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages and messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message.role == "user" %}
+        {%- if message['content'] -%}
+            {%- if message['content'] is string -%}
+                {{ '<|im_start|>user\n' + message['content'] }}
+            {%- else -%}
+                {{ '<|im_start|>user\n' }}
+                {%- for content in message['content'] -%}
+                    {%- if content['type'] == 'text' -%}
+                        {{ content['text'] }}
+                    {%- else -%}
+                        {{ '<|vision_start|><|image_pad|><|vision_end|>' }}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+            {{ '<|im_end|>\n' }}
+        {%- endif -%}
+    {%- elif (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}