{%- macro render_content(content, num_img_tokens, num_video_frames) -%} {%- if content is string -%} {{- content -}} {%- elif content is sequence -%} {%- set ns = namespace(out="", prev_was_text=false) -%} {%- for item in content -%} {%- set item_type = item.get("type") -%} {%- if item_type == "text" or item.get("text") is not none -%} {%- set text = item.get("text", "") -%} {%- if text -%} {%- if ns.prev_was_text -%} {%- set ns.out = ns.out ~ " " -%} {%- endif -%} {%- set ns.out = ns.out ~ text -%} {%- endif -%} {%- set ns.prev_was_text = text != "" -%} {%- elif item_type in ["image", "image_url"] or item.get("image") is not none or item.get("image_url") is not none -%} {%- set ns.out = ns.out ~ "" ~ ("" * num_img_tokens) ~ "" -%} {%- set ns.prev_was_text = false -%} {%- elif item_type in ["video", "video_url"] or item.get("video") is not none or item.get("video_url") is not none -%} {%- set repeat_tokens = num_img_tokens * num_video_frames -%} {%- set ns.out = ns.out ~ "" -%} {%- set ns.prev_was_text = false -%} {%- endif -%} {%- endfor -%} {{- ns.out -}} {%- endif -%} {%- endmacro -%} {%- set ns = namespace(out="", last_query_index=messages|length - 1) -%} {%- for msg in messages[::-1] -%} {%- set idx = messages|length - 1 - loop.index0 -%} {%- if msg.get("role") == "user" -%} {%- set content = msg.get("content", "") -%} {%- if not (content is string and content.startswith("") and content.endswith("")) -%} {%- set ns.last_query_index = idx -%} {%- break -%} {%- endif -%} {%- endif -%} {%- endfor -%} {%- set last_query_index = ns.last_query_index -%} {%- set num_img_tokens = num_img_tokens | default(64, true) | int -%} {%- set num_video_frames = num_video_frames | default(6, true) | int -%} {%- set start_idx = 0 -%} {%- set system_text = "" -%} {%- if messages|length > 0 and messages[0].get("role") in ["system", "developer"] -%} {%- set system_text = render_content(messages[0].get("content", ""), num_img_tokens, num_video_frames) -%} {%- set start_idx = 1 -%} {%- endif -%} {%- if tools or system_text -%} {%- set preamble_ns = namespace(text="") -%} {%- if system_text -%} {%- set preamble_ns.text = "system: " ~ system_text -%} {%- endif -%} {%- if tools -%} {%- if preamble_ns.text -%} {%- set preamble_ns.text = preamble_ns.text ~ "\n\n" -%} {%- else -%} {%- set preamble_ns.text = "system: " -%} {%- endif -%} {%- set preamble_ns.text = preamble_ns.text ~ "# Tools\n\n" ~ "You may call one or more functions to assist with the user query.\n\n" ~ "You are provided with function signatures within XML tags:\n" ~ "" -%} {%- for tool in tools -%} {%- set preamble_ns.text = preamble_ns.text ~ "\n" ~ (tool | tojson(ensure_ascii=True)) -%} {%- endfor -%} {%- set preamble_ns.text = preamble_ns.text ~ "\n\n\n" ~ "For each function call, return a json object with function name and arguments " ~ "within XML tags:\n" ~ "\n{\"name\": , \"arguments\": }\n" -%} {%- endif -%} {%- set ns.out = ns.out ~ preamble_ns.text ~ "\n\n" -%} {%- endif -%} {%- for idx in range(start_idx, messages|length) -%} {%- set message = messages[idx] -%} {%- set role = message.get("role") -%} {%- set content = message.get("content") -%} {%- if role == "user" -%} {%- set prefix_ns = namespace(value="human: ") -%} {%- if content is sequence and content is not string -%} {%- for item in content -%} {%- if item.get("type") == "text" or item.get("text") is not none -%} {%- set text = item.get("text", "") -%} {%- if text -%} {%- break -%} {%- endif -%} {%- elif item.get("type") in ["image", "image_url", "video", "video_url"] -%} {%- set prefix_ns.value = "human:" -%} {%- break -%} {%- endif -%} {%- endfor -%} {%- endif -%} {%- set ns.out = ns.out ~ prefix_ns.value ~ render_content(content, num_img_tokens, num_video_frames) ~ "" -%} {%- elif role == "assistant" -%} {%- set tool_calls = message.get("tool_calls") -%} {%- set content_text = render_content(content, num_img_tokens, num_video_frames) -%} {%- set reasoning_text = "" -%} {%- if message.get("reasoning_content") is string -%} {%- set reasoning_text = message.get("reasoning_content") -%} {%- elif "" in content_text -%} {%- set reasoning_text = content_text.split("", 1)[0].rstrip("\n").split("")[-1].lstrip("\n") -%} {%- set content_text = content_text.split("", 1)[1].lstrip("\n") -%} {%- endif -%} {%- set ns.out = ns.out ~ "assistant: " -%} {%- set include_thinking = enable_thinking is true and idx > last_query_index and (idx == messages|length - 1 or reasoning_text) -%} {%- if include_thinking -%} {%- set ns.out = ns.out ~ "\n" ~ (reasoning_text.strip() ) ~ "\n\n\n" -%} {%- endif -%} {%- set ns.out = ns.out ~ content_text -%} {%- if tool_calls -%} {%- if content_text and not ns.out.endswith("\n") -%} {%- set ns.out = ns.out ~ "\n" -%} {%- endif -%} {%- for tool_call in tool_calls -%} {%- if tool_call.get("function") is not none -%} {%- set tool_call = tool_call.get("function") -%} {%- endif -%} {%- set arguments = tool_call.get("arguments", {}) -%} {%- if arguments is string -%} {%- set arguments_json = arguments -%} {%- elif arguments is mapping -%} {%- set arguments_json = arguments | tojson(ensure_ascii=True) -%} {%- else -%} {%- set arguments_json = arguments | tojson(ensure_ascii=True) -%} {%- endif -%} {%- set ns.out = ns.out ~ "\n" ~ "{\"name\": \"" ~ tool_call.get("name", "") ~ "\", \"arguments\": " ~ arguments_json ~ "}\n" -%} {%- endfor -%} {%- endif -%} {%- if not (continue_final_message and idx == messages|length - 1) -%} {%- set ns.out = ns.out ~ "\n\n" -%} {%- endif -%} {%- elif role == "tool" -%} {%- if idx == start_idx or messages[idx - 1].get("role") != "tool" -%} {%- set ns.out = ns.out ~ "human: " -%} {%- endif -%} {%- set response_text = render_content(content, num_img_tokens, num_video_frames) -%} {%- set ns.out = ns.out ~ "\n" ~ response_text ~ "\n" -%} {%- if idx == messages|length - 1 or messages[idx + 1].get("role") != "tool" -%} {%- set ns.out = ns.out ~ "" -%} {%- endif -%} {%- endif -%} {%- endfor -%} {%- if add_generation_prompt and (messages|length == 0 or messages[-1].get("role") != "assistant") -%} {%- if enable_thinking is true -%} {%- set ns.out = ns.out ~ "assistant: \n" -%} {%- else -%} {%- set ns.out = ns.out ~ "assistant:" -%} {%- endif -%} {%- endif -%} {{- ns.out -}}