diff --git a/backend/src/core/agentscope/events/agui_codec.py b/backend/src/core/agentscope/events/agui_codec.py index b7974fc..61a5b18 100644 --- a/backend/src/core/agentscope/events/agui_codec.py +++ b/backend/src/core/agentscope/events/agui_codec.py @@ -38,9 +38,10 @@ _INTERNAL_TO_AGUI: dict[str, EventType] = { def _convert_to_agui_type(internal_type: str) -> EventType: - return _INTERNAL_TO_AGUI.get( - internal_type, EventType(internal_type.upper().replace(".", "_")) - ) + mapped = _INTERNAL_TO_AGUI.get(internal_type) + if mapped is not None: + return mapped + return EventType(internal_type.upper().replace(".", "_")) def _is_agui_event(event: dict[str, Any]) -> bool: @@ -142,32 +143,64 @@ def to_agui_wire_event(event: dict[str, Any] | BaseEvent) -> dict[str, Any]: return event internal_type = str(event.get("type", "")).strip() + + thread_id = event.get("threadId") + run_id = event.get("runId") + data = event.get("data") + + if internal_type == "text.end" and isinstance(data, dict): + text_end_payload: dict[str, Any] = { + "type": _convert_to_agui_type(internal_type).value, + } + if isinstance(thread_id, str) and thread_id: + text_end_payload["threadId"] = thread_id + if isinstance(run_id, str) and run_id: + text_end_payload["runId"] = run_id + for key in ("messageId", "workerAgentOutput"): + value = data.get(key) + if value is not None: + text_end_payload[key] = value + return text_end_payload + + if internal_type == "tool.result" and isinstance(data, dict): + tool_result_payload = { + "type": _convert_to_agui_type(internal_type).value, + } + if isinstance(thread_id, str) and thread_id: + tool_result_payload["threadId"] = thread_id + if isinstance(run_id, str) and run_id: + tool_result_payload["runId"] = run_id + for key in ("messageId", "toolCallId", "toolAgentOutput"): + value = data.get(key) + if value is not None: + tool_result_payload[key] = value + return tool_result_payload + builder = _BUILDER_MAP.get(internal_type) if builder: agui_event = builder(event) - return agui_event.model_dump(by_alias=True, exclude_none=True) + payload = agui_event.model_dump(by_alias=True, exclude_none=True) + if isinstance(thread_id, str) and thread_id: + payload["threadId"] = thread_id + if isinstance(run_id, str) and run_id: + payload["runId"] = run_id + if isinstance(data, dict): + reserved = {"type", "threadId", "runId"} + payload.update({k: v for k, v in data.items() if k not in reserved}) + return payload wire_type = _convert_to_agui_type(internal_type) payload: dict[str, Any] = { "type": wire_type.value, } - thread_id = event.get("threadId") - run_id = event.get("runId") if isinstance(thread_id, str) and thread_id: payload["threadId"] = thread_id if isinstance(run_id, str) and run_id: payload["runId"] = run_id - data = event.get("data") if isinstance(data, dict): - if internal_type == "text.end": - for key in ("messageId", "workerAgentOutput"): - value = data.get(key) - if value is not None: - payload[key] = value - return payload reserved = {"type", "threadId", "runId"} data_map = cast(dict[str, Any], data) payload.update({k: v for k, v in data_map.items() if k not in reserved}) diff --git a/backend/src/core/agentscope/events/pipeline.py b/backend/src/core/agentscope/events/pipeline.py index 40ef8be..84422d9 100644 --- a/backend/src/core/agentscope/events/pipeline.py +++ b/backend/src/core/agentscope/events/pipeline.py @@ -50,5 +50,5 @@ class AgentScopeEventPipeline: ) -> str: event_dict = to_dict(event) wire_event = self._codec.to_wire(event_dict) - await self._store.persist(wire_event) + await self._store.persist(event_dict) return await self._bus.publish(session_id=session_id, event=wire_event) diff --git a/backend/src/core/agentscope/events/store.py b/backend/src/core/agentscope/events/store.py index 53c3fea..f67120c 100644 --- a/backend/src/core/agentscope/events/store.py +++ b/backend/src/core/agentscope/events/store.py @@ -55,8 +55,8 @@ class SqlAlchemyEventStore: self._message_contexts: dict[tuple[str, str], dict[str, object]] = {} async def persist(self, event: dict[str, Any]) -> None: - event_type = str(event.get("type", "")).strip().upper() - thread_id = event.get("threadId") + event_type = str(event.get("type", "")).strip().upper().replace(".", "_") + thread_id = self._event_value(event, "threadId") if not isinstance(thread_id, str) or not thread_id: return try: @@ -124,8 +124,8 @@ class SqlAlchemyEventStore: await session.commit() def _buffer_text_delta(self, *, session_key: str, event: dict[str, Any]) -> None: - message_id = event.get("messageId") - delta = event.get("delta") + message_id = self._event_value(event, "messageId") + delta = self._event_value(event, "delta") if not isinstance(message_id, str) or not message_id: return if not isinstance(delta, str) or not delta: @@ -143,13 +143,13 @@ class SqlAlchemyEventStore: self._message_contexts.pop(key, None) def _buffer_text_context(self, *, session_key: str, event: dict[str, Any]) -> None: - message_id = event.get("messageId") + message_id = self._event_value(event, "messageId") if not isinstance(message_id, str) or not message_id: return key = (session_key, message_id) - role = event.get("role") - stage = event.get("stage") - tool_name = event.get("toolName") + role = self._event_value(event, "role") + stage = self._event_value(event, "stage") + tool_name = self._event_value(event, "toolName") context: dict[str, object] = {} if isinstance(role, str) and role: context["role"] = role @@ -168,7 +168,7 @@ class SqlAlchemyEventStore: session_repo: SessionRepository, message_repo: MessageRepository, ) -> None: - message_id_raw = event.get("messageId") + message_id_raw = self._event_value(event, "messageId") message_id = message_id_raw if isinstance(message_id_raw, str) else "" key = (str(session_id), message_id) content = self._message_buffers.get(key, "") @@ -177,26 +177,26 @@ class SqlAlchemyEventStore: context = self._message_contexts.get(key, {}) - input_tokens = self._to_int(event.get("inputTokens")) - output_tokens = self._to_int(event.get("outputTokens")) + input_tokens = self._to_int(self._event_value(event, "inputTokens")) + output_tokens = self._to_int(self._event_value(event, "outputTokens")) token_delta = input_tokens + output_tokens - cost = self._to_decimal(event.get("cost")) - latency_ms = self._to_int_or_none(event.get("latencyMs")) - run_id = event.get("runId") - model_code = event.get("model") + cost = self._to_decimal(self._event_value(event, "cost")) + latency_ms = self._to_int_or_none(self._event_value(event, "latencyMs")) + run_id = self._event_value(event, "runId") + model_code = self._event_value(event, "model") metadata: dict[str, object] = {"message_id": message_id} if isinstance(run_id, str) and run_id: metadata["run_id"] = run_id if latency_ms is not None: metadata["latency_ms"] = latency_ms - stage = event.get("stage") + stage = self._event_value(event, "stage") if not isinstance(stage, str): stage = context.get("stage") if isinstance(stage, str) and stage: metadata["stage"] = stage - worker_payload = event.get("workerAgentOutput") + worker_payload = self._event_value(event, "workerAgentOutput") if isinstance(worker_payload, dict): try: if "ui_hints" in worker_payload: @@ -264,11 +264,11 @@ class SqlAlchemyEventStore: session_repo: SessionRepository, message_repo: MessageRepository, ) -> None: - tool_name = event.get("toolName") + tool_name = self._event_value(event, "toolName") if not isinstance(tool_name, str) or not tool_name: return - raw_output = event.get("toolAgentOutput") + raw_output = self._event_value(event, "toolAgentOutput") if not isinstance(raw_output, dict): return try: @@ -276,11 +276,11 @@ class SqlAlchemyEventStore: except Exception: return - run_id = event.get("runId") + run_id = self._event_value(event, "runId") run_id_value = run_id if isinstance(run_id, str) and run_id else "" - task_id = event.get("taskId") + task_id = self._event_value(event, "taskId") task_id_value = task_id if isinstance(task_id, str) and task_id else "task" - call_id_value = event.get("callId") + call_id_value = self._event_value(event, "callId") if not isinstance(call_id_value, str) or not call_id_value: call_id_value = ( f"{run_id_value}-{task_id_value}-{uuid4().hex[:8]}" @@ -303,7 +303,7 @@ class SqlAlchemyEventStore: } if run_id_value: metadata["run_id"] = run_id_value - stage = event.get("stage") + stage = self._event_value(event, "stage") if isinstance(stage, str) and stage: metadata["stage"] = stage if task_id_value: @@ -421,6 +421,19 @@ class SqlAlchemyEventStore: return Decimal("0") return parsed if parsed >= 0 else Decimal("0") + def _event_value( + self, + event: dict[str, Any], + key: str, + default: object | None = None, + ) -> object | None: + if key in event: + return event.get(key) + data = event.get("data") + if isinstance(data, dict): + return data.get(key, default) + return default + def _sanitize_path_component(value: str) -> str: compact = re.sub(r"[^A-Za-z0-9._-]", "-", value.strip()) diff --git a/backend/src/core/agentscope/prompts/system_prompt.py b/backend/src/core/agentscope/prompts/system_prompt.py index 8d242fd..3ff7ce6 100644 --- a/backend/src/core/agentscope/prompts/system_prompt.py +++ b/backend/src/core/agentscope/prompts/system_prompt.py @@ -2,9 +2,10 @@ from __future__ import annotations import json from datetime import datetime, timezone -from typing import Any +from typing import Any, Sequence from zoneinfo import ZoneInfo, ZoneInfoNotFoundError +from ag_ui.core.types import Tool from core.agentscope.prompts.agent_prompt import ( build_agent_prompt, ) @@ -193,7 +194,7 @@ def build_system_prompt( user_context: UserContext, now_utc: datetime, extra_context: str | None = None, - tools: list[dict[str, Any]] | None = None, + tools: Sequence[Tool] | None = None, ) -> str: sections = [ _build_identity_section(), diff --git a/backend/src/core/agentscope/prompts/tool_prompt.py b/backend/src/core/agentscope/prompts/tool_prompt.py index 9a5c5b4..d1aa7cc 100644 --- a/backend/src/core/agentscope/prompts/tool_prompt.py +++ b/backend/src/core/agentscope/prompts/tool_prompt.py @@ -1,7 +1,9 @@ from __future__ import annotations import json -from typing import Any, Iterable +from typing import Iterable + +from ag_ui.core.types import Tool def _wrap_section(section: str, content: str) -> str: @@ -15,18 +17,16 @@ def _wrap_section(section: str, content: str) -> str: def build_tools_prompt( *, - tools: Iterable[dict[str, Any]], + tools: Iterable[Tool], ) -> str: lines: list[str] = [] lines.append("[Available Tools]") for item in tools: - name = item.get("name") - description = item.get("description") or "" - parameters = item.get("parameters") or {} - if not isinstance(name, str) or not name: - continue - lines.append(f"- {name}: {description}".strip()) + name = item.name + description = item.description or "" + parameters = item.parameters or {} + lines.append(f"- {name}: {description}") lines.append( " - args_schema: " + json.dumps(parameters, ensure_ascii=True, separators=(",", ":")) diff --git a/backend/src/core/agentscope/runtime/react_runner.py b/backend/src/core/agentscope/runtime/react_runner.py index b0f78b9..2a70f4e 100644 --- a/backend/src/core/agentscope/runtime/react_runner.py +++ b/backend/src/core/agentscope/runtime/react_runner.py @@ -1,16 +1,322 @@ from __future__ import annotations +import json +from collections.abc import AsyncGenerator, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from decimal import Decimal from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 from ag_ui.core.types import RunAgentInput +from agentscope.agent import ReActAgent +from agentscope.formatter import OpenAIChatFormatter +from agentscope.memory import InMemoryMemory from agentscope.message import Msg +from agentscope.model import OpenAIChatModel +from core.agentscope.events.persistence import MessageRepository, SessionRepository +from core.agentscope.prompts.system_prompt import build_system_prompt +from core.agentscope.tools.toolkit import build_stage_toolkit +from core.db.session import AsyncSessionLocal +from core.logging import get_logger +from models.agent_chat_message import AgentChatMessageRole +from models.agent_chat_session import AgentChatSessionStatus +from models.llm import Llm +from models.system_agents import SystemAgents +from schemas.agent.runtime_models import ( + RouterAgentOutput, + ToolAgentOutput, + WorkerAgentOutputLite, + resolve_worker_output_model, +) +from schemas.agent.system_agent import AgentType, SystemAgentLLMConfig +from schemas.messages.chat_message import AgentChatMessage, AgentChatMessageMetadata from schemas.user import UserContext +from services.litellm.service import LiteLLMService +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession if TYPE_CHECKING: from core.agentscope.runtime.orchestrator import PipelineLike +logger = get_logger("core.agentscope.runtime.react_runner") + + +@dataclass(frozen=True) +class SystemAgentRuntimeConfig: + agent_type: AgentType + model_code: str + llm_config: SystemAgentLLMConfig + + +@dataclass(frozen=True) +class StageExecutionResult: + message: Msg + payload: dict[str, Any] + response_metadata: dict[str, Any] + + +class _TrackingChatModel: + def __init__(self, inner: OpenAIChatModel) -> None: + self._inner = inner + self._total_input_tokens = 0 + self._total_output_tokens = 0 + self._total_latency_ms = 0 + self._cached_prompt_tokens = 0 + + @property + def stream(self) -> bool: + return self._inner.stream + + @stream.setter + def stream(self, value: bool) -> None: + self._inner.stream = value + + def __getattr__(self, name: str) -> Any: + return getattr(self._inner, name) + + async def __call__(self, *args: Any, **kwargs: Any) -> Any: + response = await self._inner(*args, **kwargs) + if isinstance(response, AsyncGenerator): + return self._track_stream(response) + self._record_usage(getattr(response, "usage", None)) + return response + + async def _track_stream( + self, response: AsyncGenerator[Any, None] + ) -> AsyncGenerator[Any, None]: + latest_usage = None + async for chunk in response: + usage = getattr(chunk, "usage", None) + if usage is not None: + latest_usage = usage + yield chunk + self._record_usage(latest_usage) + + def _record_usage(self, usage: Any) -> None: + if usage is None: + return + self._total_input_tokens += max(int(getattr(usage, "input_tokens", 0) or 0), 0) + self._total_output_tokens += max( + int(getattr(usage, "output_tokens", 0) or 0), 0 + ) + self._total_latency_ms += max( + int(round(float(getattr(usage, "time", 0) or 0) * 1000)), 0 + ) + metadata = getattr(usage, "metadata", None) + if metadata is not None: + cached_tokens = 0 + if isinstance(metadata, dict): + prompt_details = metadata.get("prompt_tokens_details") + if isinstance(prompt_details, dict): + cached_tokens = int(prompt_details.get("cached_tokens", 0) or 0) + else: + prompt_details = getattr(metadata, "prompt_tokens_details", None) + cached_tokens = int(getattr(prompt_details, "cached_tokens", 0) or 0) + self._cached_prompt_tokens += max(cached_tokens, 0) + + def usage_summary(self) -> dict[str, int]: + return { + "input_tokens": self._total_input_tokens, + "output_tokens": self._total_output_tokens, + "latency_ms": self._total_latency_ms, + "cached_prompt_tokens": self._cached_prompt_tokens, + } + + +class _PipelineStageEmitter: + def __init__( + self, + *, + pipeline: PipelineLike, + session_id: str, + run_id: str, + stage: str, + emit_text_events: bool, + emit_tool_events: bool, + ) -> None: + self._pipeline = pipeline + self._session_id = session_id + self._run_id = run_id + self._stage = stage + self._emit_text_events = emit_text_events + self._emit_tool_events = emit_tool_events + self._text_by_message_id: dict[str, str] = {} + self._emitted_tool_calls: set[str] = set() + self._emitted_tool_results: set[str] = set() + self.latest_text_message_id: str | None = None + self.latest_text: str = "" + + async def handle_print(self, *, msg: Msg, last: bool) -> None: + del last + if self._emit_tool_events: + await self._emit_tool_events_from_msg(msg) + if self._emit_text_events: + await self._emit_text_events_from_msg(msg) + + async def _emit_text_events_from_msg(self, msg: Msg) -> None: + text = msg.get_text_content(separator="") or "" + if not text: + return + message_id = str(msg.id) + previous = self._text_by_message_id.get(message_id, "") + if message_id not in self._text_by_message_id: + await self._emit( + "text.start", + { + "messageId": message_id, + "role": "assistant", + "stage": self._stage, + }, + ) + delta = text[len(previous) :] if text.startswith(previous) else text + if delta: + await self._emit( + "text.delta", + { + "messageId": message_id, + "delta": delta, + "stage": self._stage, + }, + ) + self._text_by_message_id[message_id] = text + self.latest_text_message_id = message_id + self.latest_text = text + + async def _emit_tool_events_from_msg(self, msg: Msg) -> None: + for block in msg.get_content_blocks("tool_use"): + tool_call_id = str(block.get("id", "")).strip() + tool_name = str(block.get("name", "")).strip() + if ( + not tool_call_id + or not tool_name + or tool_call_id in self._emitted_tool_calls + ): + continue + payload = { + "messageId": str(msg.id), + "toolCallId": tool_call_id, + "toolName": tool_name, + "stage": self._stage, + } + await self._emit("tool.start", payload) + await self._emit( + "tool.args", + { + **payload, + "args": block.get("input", {}), + }, + ) + await self._emit("tool.end", payload) + self._emitted_tool_calls.add(tool_call_id) + + for block in msg.get_content_blocks("tool_result"): + tool_call_id = str(block.get("id", "")).strip() + if not tool_call_id or tool_call_id in self._emitted_tool_results: + continue + tool_output = _parse_tool_agent_output(block.get("output")) + if tool_output is None: + continue + await self._emit( + "tool.result", + { + "messageId": str(msg.id), + "toolCallId": tool_call_id, + "toolName": tool_output.tool_name, + "stage": self._stage, + "toolAgentOutput": tool_output.model_dump( + mode="json", exclude_none=True + ), + }, + ) + self._emitted_tool_results.add(tool_call_id) + + async def emit_final_text_end( + self, + *, + worker_output: dict[str, Any], + response_metadata: dict[str, Any], + ) -> None: + message_id = ( + self.latest_text_message_id or f"worker-{self._run_id}-{uuid4().hex[:8]}" + ) + if self.latest_text_message_id is None and worker_output.get("answer"): + await self._emit( + "text.start", + { + "messageId": message_id, + "role": "assistant", + "stage": self._stage, + }, + ) + await self._emit( + "text.delta", + { + "messageId": message_id, + "delta": worker_output.get("answer", ""), + "stage": self._stage, + }, + ) + await self._emit( + "text.end", + { + "messageId": message_id, + "role": "assistant", + "stage": self._stage, + "workerAgentOutput": worker_output, + **response_metadata, + }, + ) + + async def _emit(self, event_type: str, data: dict[str, Any]) -> None: + await self._pipeline.emit( + session_id=self._session_id, + event={ + "type": event_type, + "threadId": self._session_id, + "runId": self._run_id, + "data": data, + }, + ) + + +class _PipelineReActAgent(ReActAgent): + def __init__( + self, *, emitter: _PipelineStageEmitter | None = None, **kwargs: Any + ) -> None: + super().__init__(**kwargs) + self._pipeline_emitter = emitter + self.disable_console_output() + + async def print(self, msg: Msg, last: bool = True, speech: Any = None) -> None: + del speech + if self._pipeline_emitter is not None: + await self._pipeline_emitter.handle_print(msg=msg, last=last) + + +def _parse_tool_agent_output(output: Any) -> ToolAgentOutput | None: + blocks = output if isinstance(output, Sequence) else [] + for block in blocks: + if not isinstance(block, dict) or block.get("type") != "text": + continue + text = block.get("text") + if not isinstance(text, str) or not text.strip(): + continue + try: + return ToolAgentOutput.model_validate(json.loads(text)) + except Exception: + return None + return None + + +def _normalize_tool_name(value: str) -> str: + return value.strip().replace(".", "_").replace("-", "_") + class AgentScopeReActRunner: + def __init__(self, *, litellm_service: LiteLLMService | None = None) -> None: + self._litellm_service = litellm_service or LiteLLMService() + async def execute( self, *, @@ -19,4 +325,367 @@ class AgentScopeReActRunner: pipeline: PipelineLike, run_input: RunAgentInput, ) -> dict[str, Any]: - raise NotImplementedError("execute method not implemented") + owner_id = UUID(user_context.id) + enabled_tool_names = self._extract_tool_names(run_input) + + async with AsyncSessionLocal() as session: + router_toolkit, worker_toolkit = self._build_toolkits( + session=session, + owner_id=owner_id, + enabled_tool_names=enabled_tool_names, + ) + + router_config = await self._load_system_agent_config( + session=session, + agent_type=AgentType.ROUTER, + ) + worker_config = await self._load_system_agent_config( + session=session, + agent_type=AgentType.WORKER, + ) + + await self._emit_step_event( + pipeline=pipeline, + run_input=run_input, + step_name="router", + event_type="step.start", + ) + router_result = await self._run_router_stage( + user_context=user_context, + context_messages=context_messages, + toolkit=router_toolkit, + run_input=run_input, + stage_config=router_config, + ) + router_output = RouterAgentOutput.model_validate(router_result.payload) + await self._persist_router_message( + session=session, + thread_id=run_input.thread_id, + run_id=run_input.run_id, + model_code=router_config.model_code, + router_output=router_output, + response_metadata=router_result.response_metadata, + ) + await session.commit() + await self._emit_step_event( + pipeline=pipeline, + run_input=run_input, + step_name="router", + event_type="step.finish", + ) + + worker_output_model = resolve_worker_output_model(router_output.ui.ui_mode) + await self._emit_step_event( + pipeline=pipeline, + run_input=run_input, + step_name="worker", + event_type="step.start", + ) + worker_result = await self._run_worker_stage( + user_context=user_context, + router_output=router_output, + toolkit=worker_toolkit, + run_input=run_input, + stage_config=worker_config, + worker_output_model=worker_output_model, + pipeline=pipeline, + ) + worker_output = worker_output_model.model_validate(worker_result.payload) + await self._emit_step_event( + pipeline=pipeline, + run_input=run_input, + step_name="worker", + event_type="step.finish", + ) + + return { + "router": router_output.model_dump(mode="json", exclude_none=True), + "worker": worker_output.model_dump(mode="json", exclude_none=True), + } + + def _build_toolkits( + self, + *, + session: AsyncSession, + owner_id: UUID, + enabled_tool_names: set[str] | None, + ) -> tuple[Any, Any]: + return ( + build_stage_toolkit( + agent_type=AgentType.ROUTER, + session=session, + owner_id=owner_id, + enabled_tool_names=enabled_tool_names, + ), + build_stage_toolkit( + agent_type=AgentType.WORKER, + session=session, + owner_id=owner_id, + enabled_tool_names=enabled_tool_names, + ), + ) + + def _extract_tool_names(self, run_input: RunAgentInput) -> set[str] | None: + raw_tools = getattr(run_input, "tools", None) + if not isinstance(raw_tools, list): + return None + selected: set[str] = set() + for item in raw_tools: + if isinstance(item, dict): + name = item.get("name") + else: + name = getattr(item, "name", None) + if isinstance(name, str) and name.strip(): + selected.add(_normalize_tool_name(name)) + return selected + + async def _load_system_agent_config( + self, + *, + session: AsyncSession, + agent_type: AgentType, + ) -> SystemAgentRuntimeConfig: + stmt = ( + select(SystemAgents, Llm) + .join(Llm, SystemAgents.llm_id == Llm.id) + .where(SystemAgents.agent_type == agent_type.value) + ) + row = (await session.execute(stmt)).one_or_none() + if row is None: + raise RuntimeError(f"system agent config not found: {agent_type.value}") + system_agent, llm = row + status = str(system_agent.status).strip().lower() + if status != "active": + raise RuntimeError(f"system agent is not active: {agent_type.value}") + return SystemAgentRuntimeConfig( + agent_type=agent_type, + model_code=llm.model_code, + llm_config=SystemAgentLLMConfig.model_validate(system_agent.config or {}), + ) + + async def _run_router_stage( + self, + *, + user_context: UserContext, + context_messages: list[Msg], + toolkit: Any, + run_input: RunAgentInput, + stage_config: SystemAgentRuntimeConfig, + ) -> StageExecutionResult: + tracking_model = self._build_model(stage_config=stage_config) + agent = self._build_agent( + agent_name="router", + system_prompt=build_system_prompt( + agent_type=AgentType.ROUTER, + user_context=user_context, + now_utc=datetime.now(timezone.utc), + tools=run_input.tools, + ), + toolkit=toolkit, + model=tracking_model, + ) + response_msg = await agent.reply( + context_messages, structured_model=RouterAgentOutput + ) + payload = RouterAgentOutput.model_validate( + response_msg.metadata or {} + ).model_dump( + mode="json", + exclude_none=True, + ) + return StageExecutionResult( + message=response_msg, + payload=payload, + response_metadata=self._litellm_service.build_usage_metadata( + model=stage_config.model_code, + usage_summary=tracking_model.usage_summary(), + ), + ) + + async def _run_worker_stage( + self, + *, + user_context: UserContext, + router_output: RouterAgentOutput, + toolkit: Any, + run_input: RunAgentInput, + stage_config: SystemAgentRuntimeConfig, + worker_output_model: type[WorkerAgentOutputLite], + pipeline: PipelineLike, + ) -> StageExecutionResult: + worker_input = self._build_worker_input_messages( + router_output=router_output, + ) + tracking_model = self._build_model(stage_config=stage_config) + emitter = _PipelineStageEmitter( + pipeline=pipeline, + session_id=run_input.thread_id, + run_id=run_input.run_id, + stage="worker", + emit_text_events=True, + emit_tool_events=True, + ) + agent = self._build_agent( + agent_name="worker", + system_prompt=build_system_prompt( + agent_type=AgentType.WORKER, + user_context=user_context, + now_utc=datetime.now(timezone.utc), + tools=run_input.tools, + ), + toolkit=toolkit, + model=tracking_model, + emitter=emitter, + ) + response_msg = await agent.reply( + worker_input, + structured_model=worker_output_model, + ) + worker_payload = worker_output_model.model_validate(response_msg.metadata or {}) + response_metadata = self._litellm_service.build_usage_metadata( + model=stage_config.model_code, + usage_summary=tracking_model.usage_summary(), + ) + await emitter.emit_final_text_end( + worker_output=worker_payload.model_dump(mode="json", exclude_none=True), + response_metadata=response_metadata, + ) + return StageExecutionResult( + message=response_msg, + payload=worker_payload.model_dump(mode="json", exclude_none=True), + response_metadata=response_metadata, + ) + + def _build_worker_input_messages( + self, + *, + router_output: RouterAgentOutput, + ) -> list[Msg]: + routing_contract = json.dumps( + router_output.model_dump(mode="json", exclude_none=True), + ensure_ascii=False, + separators=(",", ":"), + ) + routing_msg = Msg( + name="router", + role="user", + content=( + "Use the following routing contract as the execution source of truth. " + f"Do not change the routed objective:\n{routing_contract}" + ), + ) + return [routing_msg] + + def _build_model( + self, *, stage_config: SystemAgentRuntimeConfig + ) -> _TrackingChatModel: + model = OpenAIChatModel( + model_name=stage_config.model_code, + api_key=self._litellm_service.proxy_api_key, + stream=True, + client_kwargs={"base_url": self._litellm_service.proxy_base_url}, + generate_kwargs={ + "temperature": stage_config.llm_config.temperature, + "max_tokens": stage_config.llm_config.max_tokens, + "timeout": stage_config.llm_config.timeout_seconds, + }, + ) + return _TrackingChatModel(model) + + def _build_agent( + self, + *, + agent_name: str, + system_prompt: str, + toolkit: Any, + model: _TrackingChatModel, + emitter: _PipelineStageEmitter | None = None, + ) -> _PipelineReActAgent: + return _PipelineReActAgent( + name=agent_name, + sys_prompt=system_prompt, + model=model, + formatter=OpenAIChatFormatter(), + toolkit=toolkit, + memory=InMemoryMemory(), + emitter=emitter, + ) + + async def _emit_step_event( + self, + *, + pipeline: PipelineLike, + run_input: RunAgentInput, + step_name: str, + event_type: str, + ) -> None: + await pipeline.emit( + session_id=run_input.thread_id, + event={ + "type": event_type, + "threadId": run_input.thread_id, + "runId": run_input.run_id, + "data": {"stepName": step_name}, + }, + ) + + async def _persist_router_message( + self, + *, + session: AsyncSession, + thread_id: str, + run_id: str, + model_code: str, + router_output: RouterAgentOutput, + response_metadata: dict[str, Any], + ) -> None: + session_id = UUID(thread_id) + message_repo = MessageRepository(session) + session_repo = SessionRepository(session) + locked_session = await session_repo.lock_session_for_update( + session_id=session_id + ) + if locked_session is None: + raise RuntimeError("chat session not found for router persistence") + seq = int(getattr(locked_session, "message_count", 0) or 0) + 1 + metadata = AgentChatMessageMetadata( + run_id=run_id, + agent_type=AgentType.ROUTER, + router_agent_output=router_output, + ) + message_payload = AgentChatMessage( + id=uuid4(), + seq=seq, + role=AgentChatMessageRole.ASSISTANT.value, + content="", + model_code=model_code, + tool_name=None, + input_tokens=int(response_metadata.get("inputTokens", 0) or 0), + output_tokens=int(response_metadata.get("outputTokens", 0) or 0), + cost=Decimal(str(response_metadata.get("cost", 0) or 0)), + latency_ms=int(response_metadata.get("latencyMs", 0) or 0), + metadata=metadata, + timestamp=datetime.now(timezone.utc), + ) + await message_repo.append_message( + session_id=session_id, + seq=message_payload.seq, + role=AgentChatMessageRole.ASSISTANT, + content=message_payload.content, + model_code=message_payload.model_code, + tool_name=message_payload.tool_name, + metadata=metadata.model_dump(mode="json", exclude_none=True), + input_tokens=message_payload.input_tokens, + output_tokens=message_payload.output_tokens, + cost=message_payload.cost, + latency_ms=message_payload.latency_ms, + ) + await session_repo.update_runtime_state( + chat_session=locked_session, + status=AgentChatSessionStatus.RUNNING, + state_snapshot=locked_session.state_snapshot or {}, + message_delta=1, + token_delta=message_payload.input_tokens + message_payload.output_tokens, + cost_delta=message_payload.cost, + ) + await session.flush() diff --git a/backend/src/core/agentscope/runtime/ui_compiler.py b/backend/src/core/agentscope/runtime/ui_compiler.py index 7fd0274..9ecb0d0 100644 --- a/backend/src/core/agentscope/runtime/ui_compiler.py +++ b/backend/src/core/agentscope/runtime/ui_compiler.py @@ -1,634 +1,635 @@ +""" +UiCompiler - 将描述性 UiHints 编译为渲染性 UiSchemaRenderer + +设计原则: +- 机械转换: 不依赖复杂语义理解 +- 尽量无损: hints 中出现的主要内容字段尽量保留 +- 弱模板: intent 只影响默认布局，不决定字段是否丢失 +""" + from __future__ import annotations from typing import Any -from schemas.agent.runtime_models import ( - ToolAgentOutput, - ToolStatus, - WorkerAgentOutput, -) from schemas.agent.ui_hints import ( UiHintAction, - UiHintActionCopy, - UiHintActionEvent, - UiHintActionNavigation, - UiHintActionPayload, - UiHintActionTool, - UiHintActionUrl, - UiHintBlock, - UiHintCardBlock, - UiHintContainerBlock, - UiHintCustomBlock, - UiHintErrorBlock, - UiHintKvBlock, - UiHintListBlock, + UiHintActionTarget, + UiHintIcon, + UiHintIntent, + UiHintKvItem, UiHintListItem, - UiHintOperationBlock, + UiHintSection, + UiHintsPayload, UiHintStatus, UiHintTextFormat, - UiHintTextBlock, - UiHintsPayload, -) -from schemas.agent.ui_schema import ( - ActionStyle, - ContainerDirection, - CopyAction, - EventAction, - KvLayout, - LinkAction, - NavigateAction, - OperationResult, - OperationType, - PayloadAction, - SchemaType, - TextFormat, - ToolAction, - UiStatus, - build_action, - build_card_node, - build_container_node, - build_document, - build_error_node, - build_kv_node, - build_list_node, - build_operation_node, - build_text_node, ) +# ============================================================ +# Helpers +# ============================================================ -class UiCompiler: - def __init__(self, *, locale: str = "zh-CN", version: str = "1.0") -> None: - self.locale = locale - self.version = version - def compile_ui_hints_to_document( - self, - ui_hints: UiHintsPayload, - *, - schema_type: SchemaType, - doc_id: str | None = None, - timestamp: str | None = None, - fallback_status: UiStatus | None = None, - meta: dict[str, Any] | None = None, - ) -> dict[str, Any]: - nodes = [self._compile_block(block) for block in ui_hints.blocks] - if ui_hints.actions: - root_actions = [self._compile_action(action) for action in ui_hints.actions] - if nodes: - nodes = [ - build_container_node( - nodes, - node_id="ui-hints-root", - direction=ContainerDirection.VERTICAL, - actions=root_actions, - ) - ] - else: - nodes = [ - build_text_node( - "可执行操作", - node_id="ui-hints-actions-only", - actions=root_actions, - ) - ] +def _compact(value: Any) -> Any: + """递归移除 dict/list 中的 None，保留 False/0/空字符串/空列表按原样存在。""" + if isinstance(value, dict): + return {k: _compact(v) for k, v in value.items() if v is not None} + if isinstance(value, list): + return [_compact(v) for v in value] + return value - status = self._map_hint_status(ui_hints.status) - if ( - fallback_status is not None - and status == UiStatus.INFO - and not ui_hints.blocks - ): - status = fallback_status - merged_meta = dict(ui_hints.meta) - if ui_hints.title: - merged_meta.setdefault("title", ui_hints.title) - if ui_hints.description: - merged_meta.setdefault("description", ui_hints.description) - if meta: - merged_meta.update(meta) +def _non_empty(nodes: list[dict[str, Any] | None]) -> list[dict[str, Any]]: + return [node for node in nodes if node is not None] - return build_document( - status=status, - nodes=nodes, - version=ui_hints.version or self.version, - schema_type=schema_type, - doc_id=doc_id, - timestamp=timestamp, - locale=self.locale, - meta=merged_meta or None, - ) - def compile_tool_output( - self, - output: ToolAgentOutput, - *, - doc_id: str | None = None, - timestamp: str | None = None, - meta: dict[str, Any] | None = None, - ) -> dict[str, Any]: - hints = output.ui_hints or self._build_default_tool_hints(output) - if output.error is not None and not self._contains_error_block(hints.blocks): - hints = self._append_error_block( - hints, - UiHintErrorBlock( - kind="error", - errorCode=output.error.code, - message=output.error.message, - retryable=output.error.retryable, - details=self._stringify_details(output.error.details), - ), - ) +def compile_status(status: UiHintStatus) -> str: + return status.value - merged_meta = { - "toolName": output.tool_name, - "toolCallId": output.tool_call_id, - "toolCallArgs": output.tool_call_args, + +def _status_badge_needed(intent: UiHintIntent, status: UiHintStatus) -> bool: + return intent == UiHintIntent.STATUS or status != UiHintStatus.INFO + + +def _status_label(status: str) -> str: + return status.upper() + + +# ============================================================ +# Action Compilation +# ============================================================ + + +def compile_action(action: UiHintActionTarget) -> dict[str, Any]: + """ + 编译 action target。 + 关键修复: + - 使用 by_alias=True，避免 toolId/successMessage/submitTo 丢失 + """ + action_dict = action.model_dump(by_alias=True, exclude_none=True) + action_type = action_dict["type"] + + if action_type == "navigation": + result: dict[str, Any] = { + "type": "navigation", + "path": action_dict["path"], } - if meta: - merged_meta.update(meta) + if "params" in action_dict: + result["params"] = action_dict["params"] + return result - return self.compile_ui_hints_to_document( - hints, - schema_type=SchemaType.TOOL_RESULT, - doc_id=doc_id, - timestamp=timestamp, - fallback_status=self._map_tool_status( - output.status, has_error=output.error is not None - ), - meta=merged_meta, + if action_type == "url": + return { + "type": "url", + "url": action_dict["url"], + "target": action_dict.get("target", "_blank"), + } + + if action_type == "event": + result = { + "type": "event", + "event": action_dict["event"], + } + if "payload" in action_dict: + result["payload"] = action_dict["payload"] + return result + + if action_type == "tool": + result = { + "type": "tool", + "toolId": action_dict["toolId"], + } + if "params" in action_dict: + result["params"] = action_dict["params"] + return result + + if action_type == "copy": + result = { + "type": "copy", + "content": action_dict["content"], + } + if "successMessage" in action_dict: + result["successMessage"] = action_dict["successMessage"] + return result + + if action_type == "payload": + result = { + "type": "payload", + "payload": action_dict["payload"], + } + if "submitTo" in action_dict: + result["submitTo"] = action_dict["submitTo"] + return result + + raise ValueError(f"Unknown action type: {action_type}") + + +def compile_button(action: UiHintAction) -> dict[str, Any]: + return _compact( + { + "type": "button", + "label": action.label, + "style": action.style.value if action.style else "primary", + "disabled": action.disabled, + "action": compile_action(action.action), + } + ) + + +# ============================================================ +# Small Node Compilation +# ============================================================ + + +def compile_icon_spec(icon: UiHintIcon) -> dict[str, Any]: + return _compact( + { + "source": icon.source.value, + "value": icon.value, + "color": icon.color, + "size": icon.size, + } + ) + + +def compile_icon_node(icon: UiHintIcon) -> dict[str, Any]: + return _compact( + { + "type": "icon", + "source": icon.source.value, + "value": icon.value, + "color": icon.color, + "size": icon.size, + } + ) + + +def compile_text( + content: str, + *, + role: str = "body", + format: UiHintTextFormat | str = UiHintTextFormat.PLAIN, + status: str | None = None, +) -> dict[str, Any]: + fmt = format.value if isinstance(format, UiHintTextFormat) else format + return _compact( + { + "type": "text", + "content": content, + "format": fmt, + "role": role, + "status": status, + } + ) + + +def compile_badge(label: str, status: str) -> dict[str, Any]: + return { + "type": "badge", + "label": label, + "status": status, + } + + +def compile_kv_item(item: UiHintKvItem) -> dict[str, Any]: + return _compact( + { + "key": item.key, + "label": item.label, + "value": item.value, + "copyable": item.copyable, + } + ) + + +def compile_kv(items: list[UiHintKvItem], columns: int = 1) -> dict[str, Any]: + return { + "type": "kv", + "items": [compile_kv_item(i) for i in items], + "columns": columns, + } + + +def compile_divider() -> dict[str, Any]: + return {"type": "divider", "inset": 0} + + +# ============================================================ +# Layout Compilation +# ============================================================ + + +def compile_stack( + children: list[dict[str, Any]], + *, + direction: str = "vertical", + gap: int = 12, + appearance: str = "plain", + align: str | None = None, + justify: str | None = None, + wrap: bool | None = None, + status: str | None = None, + node_id: str | None = None, +) -> dict[str, Any]: + return _compact( + { + "type": "stack", + "id": node_id, + "direction": direction, + "gap": gap, + "appearance": appearance, + "align": align, + "justify": justify, + "wrap": wrap, + "status": status, + "children": children, + } + ) + + +def compile_grid( + children: list[dict[str, Any]], + *, + columns: int, + gap: int = 12, + appearance: str = "plain", + status: str | None = None, + node_id: str | None = None, +) -> dict[str, Any]: + return _compact( + { + "type": "grid", + "id": node_id, + "columns": columns, + "gap": gap, + "appearance": appearance, + "status": status, + "children": children, + } + ) + + +def compile_card( + children: list[dict[str, Any]], *, status: str | None = None +) -> dict[str, Any]: + return compile_stack( + children, + direction="vertical", + gap=12, + appearance="card", + status=status, + ) + + +def compile_section( + *, + title: str | None = None, + description: str | None = None, + icon: UiHintIcon | None = None, + children: list[dict[str, Any]], + status: str | None = None, +) -> dict[str, Any]: + section_children: list[dict[str, Any]] = [] + + header_row_children: list[dict[str, Any]] = [] + if icon: + header_row_children.append(compile_icon_node(icon)) + if title: + header_row_children.append(compile_text(title, role="title")) + + if header_row_children: + section_children.append( + compile_stack( + header_row_children, + direction="horizontal", + gap=8, + align="center", + ) ) - def compile_worker_output( - self, - output: WorkerAgentOutput, - *, - doc_id: str | None = None, - timestamp: str | None = None, - meta: dict[str, Any] | None = None, - ) -> dict[str, Any]: - output_ui_hints = getattr(output, "ui_hints", None) - hints = output_ui_hints or self._build_default_worker_hints(output) - output_error = getattr(output, "error", None) - if output_error is not None and not self._contains_error_block(hints.blocks): - hints = self._append_error_block( - hints, - UiHintErrorBlock( - kind="error", - errorCode=output_error.code, - message=output_error.message, - retryable=output_error.retryable, - details=self._stringify_details(output_error.details), - ), + if description: + section_children.append(compile_text(description, role="caption")) + + section_children.extend(children) + + return compile_stack( + section_children, + direction="vertical", + gap=12, + appearance="section", + status=status, + ) + + +# ============================================================ +# Block Compilation +# ============================================================ + + +def compile_action_row(actions: list[UiHintAction]) -> dict[str, Any] | None: + if not actions: + return None + buttons = [compile_button(a) for a in actions] + return compile_stack( + buttons, + direction="horizontal", + gap=8, + align="center", + wrap=True, + ) + + +def compile_list_item(item: UiHintListItem) -> dict[str, Any]: + lead_children: list[dict[str, Any]] = [compile_text(item.title, role="body")] + + if item.subtitle: + lead_children.append(compile_text(item.subtitle, role="caption")) + if item.description: + lead_children.append(compile_text(item.description, role="caption")) + + lead_block = compile_stack( + lead_children, + direction="vertical", + gap=4, + ) + + main_row_children: list[dict[str, Any]] = [] + if item.icon: + main_row_children.append(compile_icon_node(item.icon)) + main_row_children.append(lead_block) + + row = compile_stack( + main_row_children, + node_id=item.id, + direction="horizontal", + gap=8, + align="center", + ) + + trailing_children: list[dict[str, Any]] = [] + if item.status: + trailing_children.append( + compile_badge( + label=_status_label(item.status.value), + status=item.status.value, ) - - merged_meta = {"resultType": output.result_type.value} - if meta: - merged_meta.update(meta) - - return self.compile_ui_hints_to_document( - hints, - schema_type=SchemaType.AGENT_RESPONSE, - doc_id=doc_id, - timestamp=timestamp, - fallback_status=self._map_run_status(output), - meta=merged_meta, ) - def _compile_block(self, block: UiHintBlock) -> dict[str, Any]: - if isinstance(block, UiHintTextBlock): - return build_text_node( - block.content, - node_id=block.id, - format=self._map_text_format(block.format.value), - actions=self._compile_actions(block.actions), + if trailing_children: + header = compile_stack( + [row, compile_stack(trailing_children, direction="horizontal", gap=8)], + direction="horizontal", + gap=8, + justify="space-between", + align="center", + ) + else: + header = row + + children: list[dict[str, Any]] = [header] + action_row = compile_action_row(item.actions) + if action_row: + children.append(action_row) + + return compile_stack( + children, + direction="vertical", + gap=8, + appearance="card" if item.actions or item.description else "plain", + status=item.status.value if item.status else None, + node_id=item.id, + ) + + +def compile_list_block(items: list[UiHintListItem]) -> dict[str, Any]: + return compile_stack( + [compile_list_item(item) for item in items], + direction="vertical", + gap=8, + ) + + +def compile_section_block( + section: UiHintSection, default_status: str +) -> dict[str, Any]: + """ + 修复点: + - 不再先把 title/description 塞进 children 再切片 + - 避免 description 重复输出 + """ + body_children: list[dict[str, Any]] = [] + + if section.content: + body_children.append( + compile_text( + section.content, + role="body", + format=section.content_format, ) - if isinstance(block, UiHintCardBlock): - return build_card_node( - node_id=block.id, - title=block.title, - description=block.description, - status=self._map_optional_status(block.status), - children=[self._compile_block(child) for child in block.children], - actions=self._compile_actions(block.actions), - ) - if isinstance(block, UiHintKvBlock): - return build_kv_node( - pairs=[pair.model_dump(exclude_none=True) for pair in block.pairs], - node_id=block.id, - title=block.title, - description=block.description, - status=self._map_optional_status(block.status), - layout=self._map_kv_layout(block.layout.value), - actions=self._compile_actions(block.actions), - ) - if isinstance(block, UiHintListBlock): - list_items: list[dict[str, Any]] = [] - for item in block.items: - dumped = item.model_dump( - by_alias=True, - exclude_none=True, - mode="json", - ) - if item.actions: - dumped["actions"] = [ - self._compile_action(action) for action in item.actions - ] - list_items.append(dumped) - return build_list_node( - items=list_items, - node_id=block.id, - title=block.title, - description=block.description, - status=self._map_optional_status(block.status), - pagination=block.pagination.model_dump(by_alias=True) - if block.pagination - else None, - empty_text=block.empty_text, - actions=self._compile_actions(block.actions), - ) - if isinstance(block, UiHintOperationBlock): - return build_operation_node( - operation=self._map_operation_type(block.operation.value), - result=self._map_operation_result(block.result.value), - node_id=block.id, - title=block.title, - description=block.description, - status=self._map_optional_status(block.status), - message=block.message, - affected_count=block.affected_count, - details=block.details, - actions=self._compile_actions(block.actions), - ) - if isinstance(block, UiHintErrorBlock): - return build_error_node( - error_code=block.error_code, - message=block.message, - node_id=block.id, - title=block.title, - details=block.details, - retryable=block.retryable, - suggestions=block.suggestions, - actions=self._compile_actions(block.actions), - ) - if isinstance(block, UiHintContainerBlock): - return build_container_node( - children=[self._compile_block(child) for child in block.children], - direction=self._map_container_direction(block.direction.value), - node_id=block.id, - gap=block.gap, - actions=self._compile_actions(block.actions), - ) - if isinstance(block, UiHintCustomBlock): - return build_card_node( - node_id=block.id, - title=block.title or "自定义内容", - description=block.description or block.renderer_key, - status=self._map_optional_status(block.status), - children=[ - build_kv_node( - pairs=self._dict_to_kv_pairs(block.payload), - node_id=f"{block.id or 'custom'}-payload", - title="payload", - ) + ) + + if section.items: + body_children.append(compile_kv(section.items)) + + if section.list_items: + body_children.append(compile_list_block(section.list_items)) + + action_row = compile_action_row(section.actions) + if action_row: + body_children.append(action_row) + + if section.title or section.description or section.icon: + return compile_section( + title=section.title, + description=section.description, + icon=section.icon, + children=body_children, + status=default_status, + ) + + return compile_stack( + body_children, + direction="vertical", + gap=12, + ) + + +# ============================================================ +# Top-level Compilation +# ============================================================ + + +def compile_header(hints: UiHintsPayload) -> dict[str, Any] | None: + status = compile_status(hints.status) + + title_row_children: list[dict[str, Any]] = [] + if hints.icon: + title_row_children.append(compile_icon_node(hints.icon)) + if hints.title: + title_row_children.append(compile_text(hints.title, role="title")) + + right_children: list[dict[str, Any]] = [] + if _status_badge_needed(hints.intent, hints.status): + right_children.append(compile_badge(_status_label(status), status)) + + header_children: list[dict[str, Any]] = [] + + if title_row_children and right_children: + header_children.append( + compile_stack( + [ + compile_stack( + title_row_children, + direction="horizontal", + gap=8, + align="center", + ), + compile_stack( + right_children, + direction="horizontal", + gap=8, + align="center", + ), ], - extensions={ - "rendererKey": block.renderer_key, - "payload": block.payload, - }, - actions=self._compile_actions(block.actions), + direction="horizontal", + gap=8, + justify="space-between", + align="center", + ) + ) + elif title_row_children: + header_children.append( + compile_stack( + title_row_children, + direction="horizontal", + gap=8, + align="center", + ) + ) + elif right_children: + header_children.append( + compile_stack( + right_children, + direction="horizontal", + gap=8, + align="center", ) - return build_error_node( - error_code="UI_HINT_BLOCK_UNSUPPORTED", - message="Unsupported ui hint block", - details=str(type(block)), ) - def _compile_actions( - self, actions: list[UiHintAction] - ) -> list[dict[str, Any]] | None: - if not actions: - return None - return [self._compile_action(action) for action in actions] + if hints.description: + header_children.append(compile_text(hints.description, role="caption")) - def _compile_action(self, action: UiHintAction) -> dict[str, Any]: - target = action.action - payload: ( - NavigateAction - | LinkAction - | EventAction - | ToolAction - | CopyAction - | PayloadAction - ) - if isinstance(target, UiHintActionNavigation): - payload = { - "type": "navigation", - "path": target.path, - } - if target.params is not None: - payload["params"] = target.params - elif isinstance(target, UiHintActionUrl): - payload = { - "type": "url", - "url": target.url, - } - if target.target is not None: - payload["target"] = target.target - elif isinstance(target, UiHintActionEvent): - payload = { - "type": "event", - "event": target.event, - } - if target.payload is not None: - payload["payload"] = target.payload - elif isinstance(target, UiHintActionTool): - payload = { - "type": "tool", - "toolId": target.tool_id, - } - if target.params is not None: - payload["params"] = target.params - elif isinstance(target, UiHintActionCopy): - payload = { - "type": "copy", - "content": target.content, - } - if target.success_message is not None: - payload["successMessage"] = target.success_message - elif isinstance(target, UiHintActionPayload): - payload = { - "type": "payload", - "payload": target.payload, - } - if target.submit_to is not None: - payload["submitTo"] = target.submit_to - else: - payload = {"type": "event", "event": "ui.hints.unsupported_action"} - - return build_action( - action_id=action.id or f"action-{action.label}", - label=action.label, - action=payload, - style=self._map_action_style(action.style), - disabled=action.disabled, - confirm=( - action.confirm.model_dump(by_alias=True, exclude_none=True) - if action.confirm - else None - ), - ) - - def _map_action_style(self, style: Any) -> ActionStyle | None: - if style is None: - return None - value = style.value if hasattr(style, "value") else str(style) - if value == "primary": - return ActionStyle.PRIMARY - if value == "secondary": - return ActionStyle.SECONDARY - if value == "ghost": - return ActionStyle.GHOST - if value == "danger": - return ActionStyle.DANGER + if not header_children: return None - def _map_hint_status(self, status: UiHintStatus) -> UiStatus: - return UiStatus(status.value) + return compile_stack( + header_children, + direction="vertical", + gap=8, + ) - def _map_optional_status(self, status: UiHintStatus | None) -> UiStatus | None: - if status is None: - return None - return self._map_hint_status(status) - def _map_text_format(self, fmt: str) -> TextFormat: - if fmt == "markdown": - return TextFormat.MARKDOWN - return TextFormat.PLAIN +def compile_body_blocks(hints: UiHintsPayload) -> list[dict[str, Any]]: + blocks: list[dict[str, Any]] = [] - def _map_kv_layout(self, layout: str) -> KvLayout: - if layout == "horizontal": - return KvLayout.HORIZONTAL - if layout == "grid": - return KvLayout.GRID - return KvLayout.VERTICAL - - def _map_operation_type(self, operation: str) -> OperationType: - if operation == "create": - return OperationType.CREATE - if operation == "update": - return OperationType.UPDATE - if operation == "delete": - return OperationType.DELETE - return OperationType.EXECUTE - - def _map_operation_result(self, result: str) -> OperationResult: - if result == "failure": - return OperationResult.FAILURE - if result == "partial": - return OperationResult.PARTIAL - return OperationResult.SUCCESS - - def _map_container_direction(self, direction: str) -> ContainerDirection: - if direction == "horizontal": - return ContainerDirection.HORIZONTAL - return ContainerDirection.VERTICAL - - def _map_tool_status(self, status: ToolStatus, *, has_error: bool) -> UiStatus: - if has_error: - return UiStatus.ERROR - if status == ToolStatus.SUCCESS: - return UiStatus.SUCCESS - if status == ToolStatus.PARTIAL: - return UiStatus.WARNING - return UiStatus.ERROR - - def _map_run_status(self, output: WorkerAgentOutput) -> UiStatus: - if output.error is not None: - return UiStatus.ERROR - if output.status.value == "success": - return UiStatus.SUCCESS - if output.status.value == "partial_success": - return UiStatus.WARNING - return UiStatus.ERROR - - def _build_default_tool_hints(self, output: ToolAgentOutput) -> UiHintsPayload: - blocks: list[UiHintBlock] = [ - UiHintTextBlock( - kind="text", - id=f"tool-{output.tool_call_id}-summary", - content=output.result_summary, - format=UiHintTextFormat.MARKDOWN, + if hints.body: + blocks.append( + compile_text( + hints.body, + role="body", + format=hints.body_format, + status=compile_status(hints.status) + if hints.intent == UiHintIntent.STATUS + else None, ) - ] - status = UiHintStatus.INFO - if output.status == ToolStatus.SUCCESS: - status = UiHintStatus.SUCCESS - elif output.status == ToolStatus.PARTIAL: - status = UiHintStatus.WARNING - elif output.status == ToolStatus.FAILURE: - status = UiHintStatus.ERROR - return UiHintsPayload(status=status, blocks=blocks) - - def _build_default_worker_hints(self, output: WorkerAgentOutput) -> UiHintsPayload: - blocks: list[UiHintBlock] = [ - UiHintTextBlock( - kind="text", - id="worker-answer", - content=output.answer, - format=UiHintTextFormat.MARKDOWN, - ) - ] - - if output.key_points: - blocks.append( - UiHintListBlock( - kind="list", - id="worker-key-points", - title="关键点", - items=[ - UiHintListItem(id=f"kp-{idx}", title=key_point) - for idx, key_point in enumerate(output.key_points, start=1) - ], - emptyText="暂无关键点", - ) - ) - - if output.suggested_actions: - blocks.append( - UiHintListBlock( - kind="list", - id="worker-suggested-actions", - title="后续建议", - items=[ - UiHintListItem(id=f"sa-{idx}", title=action) - for idx, action in enumerate(output.suggested_actions, start=1) - ], - emptyText="暂无建议", - ) - ) - - status = UiHintStatus.INFO - if output.status.value == "success": - status = UiHintStatus.SUCCESS - elif output.status.value == "partial_success": - status = UiHintStatus.WARNING - elif output.status.value == "failed": - status = UiHintStatus.ERROR - return UiHintsPayload(status=status, blocks=blocks) - - def _contains_error_block(self, blocks: list[UiHintBlock]) -> bool: - for block in blocks: - if isinstance(block, UiHintErrorBlock): - return True - if isinstance( - block, (UiHintCardBlock, UiHintContainerBlock) - ) and self._contains_error_block(block.children): - return True - return False - - def _append_error_block( - self, - hints: UiHintsPayload, - error_block: UiHintErrorBlock, - ) -> UiHintsPayload: - merged_blocks = [*hints.blocks, error_block] - return UiHintsPayload( - version=hints.version, - status=UiHintStatus.ERROR, - title=hints.title, - description=hints.description, - blocks=merged_blocks, - actions=hints.actions, - meta=hints.meta, ) - def _dict_to_kv_pairs(self, payload: dict[str, Any]) -> list[dict[str, Any]]: - pairs: list[dict[str, Any]] = [] - for key, value in payload.items(): - if isinstance(value, (str, int, bool)) or value is None: - pairs.append( - { - "key": key, - "label": key, - "value": value, - "copyable": isinstance(value, str), - } - ) - else: - pairs.append( - { - "key": key, - "label": key, - "value": str(value), - "copyable": False, - } - ) - return pairs + if hints.items: + blocks.append(compile_kv(hints.items)) - def _stringify_details(self, details: dict[str, Any] | None) -> str | None: - if not details: - return None - pairs = [f"{key}={value}" for key, value in details.items()] - return "; ".join(pairs) + if hints.list_items: + blocks.append(compile_list_block(hints.list_items)) + + if hints.sections: + blocks.extend( + [ + compile_section_block(section, compile_status(hints.status)) + for section in hints.sections + ] + ) + + return blocks -UiBuilder = UiCompiler +def compile_footer(hints: UiHintsPayload) -> dict[str, Any] | None: + return compile_action_row(hints.actions) -def compile_ui_hints_to_schema( - ui_hints: UiHintsPayload, - *, - schema_type: SchemaType = SchemaType.TOOL_RESULT, - locale: str = "zh-CN", - version: str = "1.0", - doc_id: str | None = None, - timestamp: str | None = None, - fallback_status: UiStatus | None = None, - meta: dict[str, Any] | None = None, -) -> dict[str, Any]: - compiler = UiCompiler(locale=locale, version=version) - return compiler.compile_ui_hints_to_document( - ui_hints, - schema_type=schema_type, - doc_id=doc_id, - timestamp=timestamp, - fallback_status=fallback_status, - meta=meta, +def _root_appearance(intent: UiHintIntent) -> str: + if intent in {UiHintIntent.DATA, UiHintIntent.STATUS, UiHintIntent.MIXED}: + return "card" + if intent == UiHintIntent.FORM: + return "section" + return "plain" + + +def _root_gap(intent: UiHintIntent) -> int: + if intent == UiHintIntent.FORM: + return 16 + return 12 + + +def compile_root(hints: UiHintsPayload) -> dict[str, Any]: + """ + intent 只影响默认布局风格，不决定字段是否保留。 + """ + children = _non_empty( + [ + compile_header(hints), + *compile_body_blocks(hints), + compile_footer(hints), + ] + ) + + if not children: + children = [compile_text("No content", role="body")] + + return compile_stack( + children, + direction="vertical", + gap=_root_gap(hints.intent), + appearance=_root_appearance(hints.intent), + status=compile_status(hints.status), ) -def build_tool_ui_schema( - output: ToolAgentOutput, - *, - locale: str = "zh-CN", - version: str = "1.0", - doc_id: str | None = None, - timestamp: str | None = None, - meta: dict[str, Any] | None = None, -) -> dict[str, Any]: - compiler = UiCompiler(locale=locale, version=version) - return compiler.compile_tool_output( - output, - doc_id=doc_id, - timestamp=timestamp, - meta=meta, - ) +# ============================================================ +# Public API +# ============================================================ -def build_worker_ui_schema( - output: WorkerAgentOutput, - *, - locale: str = "zh-CN", - version: str = "1.0", - doc_id: str | None = None, - timestamp: str | None = None, - meta: dict[str, Any] | None = None, -) -> dict[str, Any]: - compiler = UiCompiler(locale=locale, version=version) - return compiler.compile_worker_output( - output, - doc_id=doc_id, - timestamp=timestamp, - meta=meta, - ) +def compile(hints: UiHintsPayload) -> dict[str, Any]: + """ + 将描述性 UiHints 编译为渲染性 UiSchemaRenderer。 + + 保证: + - title / description / body / items / listItems / sections / actions / icon 尽量保留 + - intent 只影响默认包装风格 + - meta 中常用 requestId/toolId/traceId/userId 会透传 + """ + root = compile_root(hints) + + meta_keys = ("requestId", "toolId", "traceId", "userId") + meta = {k: hints.meta.get(k) for k in meta_keys if hints.meta.get(k) is not None} + + result: dict[str, Any] = { + "version": "2.0", + "locale": "zh-CN", + "status": compile_status(hints.status), + "theme": "default", + "root": root, + } + + if meta: + result["meta"] = meta + + return _compact(result) diff --git a/backend/src/core/agentscope/tools/custom/user_lookup.py b/backend/src/core/agentscope/tools/custom/user_lookup.py index 8731bbc..85e56c7 100644 --- a/backend/src/core/agentscope/tools/custom/user_lookup.py +++ b/backend/src/core/agentscope/tools/custom/user_lookup.py @@ -23,9 +23,8 @@ from schemas.agent.ui_hints import ( UiHintAction, UiHintActionCopy, UiHintActionStyle, - UiHintErrorBlock, - UiHintKeyValuePair, - UiHintKvBlock, + UiHintIntent, + UiHintKvItem, UiHintStatus, UiHintsPayload, ) @@ -54,18 +53,10 @@ def _lookup_error_output( update={ "tool_call_args": tool_call_args, "ui_hints": UiHintsPayload( + intent=UiHintIntent.STATUS, status=UiHintStatus.ERROR, title="用户查找失败", - description=message, - blocks=[ - UiHintErrorBlock( - kind="error", - title="查找失败", - errorCode=code, - message=message, - retryable=retryable, - ) - ], + body=message, ), } ) @@ -78,28 +69,15 @@ def _lookup_success_hints(resolved: dict[str, Any]) -> UiHintsPayload: username = str(resolved.get("username") or "") matched_by = str(resolved.get("matchedBy") or "") return UiHintsPayload( + intent=UiHintIntent.DATA, status=UiHintStatus.SUCCESS, title="用户信息", description=f"匹配方式: {matched_by}", - blocks=[ - UiHintKvBlock( - kind="kv", - title="查找结果", - pairs=[ - UiHintKeyValuePair( - key="user_id", label="用户ID", value=user_id, copyable=True - ), - UiHintKeyValuePair( - key="email", label="邮箱", value=email, copyable=True - ), - UiHintKeyValuePair( - key="username", label="用户名", value=username or "-" - ), - UiHintKeyValuePair( - key="matched_by", label="匹配方式", value=matched_by - ), - ], - ) + items=[ + UiHintKvItem(key="user_id", label="用户ID", value=user_id, copyable=True), + UiHintKvItem(key="email", label="邮箱", value=email, copyable=True), + UiHintKvItem(key="username", label="用户名", value=username or "-"), + UiHintKvItem(key="matched_by", label="匹配方式", value=matched_by), ], actions=[ UiHintAction( diff --git a/backend/src/core/agentscope/tools/toolkit.py b/backend/src/core/agentscope/tools/toolkit.py index 9487645..432bb68 100644 --- a/backend/src/core/agentscope/tools/toolkit.py +++ b/backend/src/core/agentscope/tools/toolkit.py @@ -18,6 +18,7 @@ from core.agentscope.tools.tool_config import ( ) from core.agentscope.tools.tool_middleware import register_tool_middlewares from sqlalchemy.ext.asyncio import AsyncSession +from schemas.agent.system_agent import AgentType TOOL_FUNCTIONS: dict[str, Any] = { "calendar_read": calendar_read, @@ -27,9 +28,9 @@ TOOL_FUNCTIONS: dict[str, Any] = { } -STAGE_TO_GROUPS: dict[str, set[ToolGroup]] = { - "router": {ToolGroup.READ}, - "worker": {ToolGroup.READ, ToolGroup.WRITE}, +AGENT_TYPE_TO_GROUPS: dict[AgentType, set[ToolGroup]] = { + AgentType.ROUTER: {ToolGroup.READ}, + AgentType.WORKER: {ToolGroup.READ, ToolGroup.WRITE}, } @@ -61,7 +62,6 @@ def build_toolkit( groups: set[ToolGroup] | None = None, enabled_tool_names: set[str] | None = None, enable_hitl: bool | None = None, - enable_approval_layer: bool = True, ): toolkit = Toolkit() enabled_names = _resolve_enabled_tools( @@ -85,7 +85,7 @@ def build_toolkit( preset_kwargs=preset_kwargs, ) - approval_enabled = enable_approval_layer if enable_hitl is None else enable_hitl + approval_enabled = enable_hitl if enable_hitl is not None else True if approval_enabled: register_tool_middlewares(toolkit=toolkit, config_by_name=TOOL_CONFIGS) @@ -94,20 +94,26 @@ def build_toolkit( def build_stage_toolkit( *, - stage: str, + agent_type: AgentType, session: AsyncSession, owner_id: UUID, + enabled_tool_names: set[str] | None = None, enable_hitl: bool | None = None, - enable_approval_layer: bool = True, ): - groups = STAGE_TO_GROUPS.get(stage) + groups = AGENT_TYPE_TO_GROUPS.get(agent_type) if groups is None: - raise ValueError(f"unknown stage: {stage}") + raise ValueError(f"unknown agent_type: {agent_type}") + + stage_enabled_names = resolve_tool_names_by_groups(set(groups)) + selected_names = ( + stage_enabled_names + if enabled_tool_names is None + else stage_enabled_names | set(enabled_tool_names) + ) return build_toolkit( session=session, owner_id=owner_id, - groups=set(groups), + enabled_tool_names=selected_names, enable_hitl=enable_hitl, - enable_approval_layer=enable_approval_layer, ) diff --git a/backend/src/core/agentscope/tools/utils/calendar_ui.py b/backend/src/core/agentscope/tools/utils/calendar_ui.py index f5263bb..8b5fce2 100644 --- a/backend/src/core/agentscope/tools/utils/calendar_ui.py +++ b/backend/src/core/agentscope/tools/utils/calendar_ui.py @@ -12,17 +12,10 @@ from schemas.agent.ui_hints import ( UiHintAction, UiHintActionNavigation, UiHintActionStyle, - UiHintErrorBlock, - UiHintKeyValuePair, - UiHintKvBlock, - UiHintListBlock, + UiHintIntent, + UiHintKvItem, UiHintListItem, - UiHintOperationBlock, - UiHintOperationResult, - UiHintOperationType, UiHintStatus, - UiHintTextBlock, - UiHintTextFormat, UiHintsPayload, ) @@ -40,18 +33,10 @@ def calendar_error_output( retryable: bool, ) -> ToolResponse: ui_hints = UiHintsPayload( + intent=UiHintIntent.STATUS, status=UiHintStatus.ERROR, title="日历操作失败", - description=message, - blocks=[ - UiHintErrorBlock( - kind="error", - title="操作失败", - errorCode=code, - message=message, - retryable=retryable, - ) - ], + body=message, ) output = build_error_output( tool_name=tool_name, @@ -84,29 +69,17 @@ def calendar_read_hints( for event in events ] return UiHintsPayload( + intent=UiHintIntent.LIST, status=UiHintStatus.SUCCESS, title="日程列表", description=f"共 {total} 个日程", - blocks=[ - UiHintKvBlock( - kind="kv", - title="分页信息", - pairs=[ - UiHintKeyValuePair(key="total", label="总数", value=total), - UiHintKeyValuePair(key="page", label="当前页", value=page), - UiHintKeyValuePair(key="page_size", label="每页", value=page_size), - UiHintKeyValuePair( - key="total_pages", label="总页数", value=total_pages - ), - ], - ), - UiHintListBlock( - kind="list", - title="日程项", - items=event_items, - emptyText="当前没有日程", - ), + items=[ + UiHintKvItem(key="total", label="总数", value=total), + UiHintKvItem(key="page", label="当前页", value=page), + UiHintKvItem(key="page_size", label="每页", value=page_size), + UiHintKvItem(key="total_pages", label="总页数", value=total_pages), ], + list_items=event_items, actions=[ UiHintAction( label="打开日历", @@ -125,65 +98,38 @@ def calendar_write_hints( event: dict[str, Any] | None, event_id: str | None, ) -> UiHintsPayload: - operation_type = UiHintOperationType.EXECUTE - if operation == "create": - operation_type = UiHintOperationType.CREATE - elif operation == "update": - operation_type = UiHintOperationType.UPDATE - elif operation == "delete": - operation_type = UiHintOperationType.DELETE + kv_items: list[UiHintKvItem] = [] - blocks: list[Any] = [ - UiHintOperationBlock( - kind="operation", - title="日历写入结果", - operation=operation_type, - result=UiHintOperationResult.SUCCESS, - message=message, - affectedCount=1, - ) - ] if event: - blocks.append( - UiHintKvBlock( - kind="kv", - title="日程详情", - pairs=[ - UiHintKeyValuePair( - key="event_id", - label="日程ID", - value=str(event.get("id") or ""), - copyable=True, - ), - UiHintKeyValuePair( - key="title", - label="标题", - value=str(event.get("title") or ""), - copyable=True, - ), - UiHintKeyValuePair( - key="start_at", - label="开始时间", - value=str(event.get("startAt") or ""), - copyable=True, - ), - ], - ) - ) + kv_items = [ + UiHintKvItem( + key="event_id", + label="日程ID", + value=str(event.get("id") or ""), + copyable=True, + ), + UiHintKvItem( + key="title", + label="标题", + value=str(event.get("title") or ""), + copyable=True, + ), + UiHintKvItem( + key="start_at", + label="开始时间", + value=str(event.get("startAt") or ""), + copyable=True, + ), + ] elif event_id: - blocks.append( - UiHintTextBlock( - kind="text", - content=f"目标日程 ID: {event_id}", - format=UiHintTextFormat.PLAIN, - ) - ) + message = f"目标日程 ID: {event_id}\n{message}" return UiHintsPayload( + intent=UiHintIntent.STATUS, status=UiHintStatus.SUCCESS, title="日历操作完成", - description=message, - blocks=blocks, + body=message, + items=kv_items if kv_items else None, actions=[ UiHintAction( label="查看日历", @@ -203,36 +149,17 @@ def calendar_share_hints( permission_text = ( ", ".join([k for k, v in permission.items() if v is True]) or "按邀请人单独设置" ) + return UiHintsPayload( + intent=UiHintIntent.STATUS, status=UiHintStatus.SUCCESS, title="日程已分享", description=f"已邀请 {len(invited)} 人", - blocks=[ - UiHintOperationBlock( - kind="operation", - title="分享结果", - operation=UiHintOperationType.EXECUTE, - result=UiHintOperationResult.SUCCESS, - message=f"已邀请 {len(invited)} 人", - affectedCount=len(invited), - ), - UiHintKvBlock( - kind="kv", - title="分享信息", - pairs=[ - UiHintKeyValuePair( - key="event_id", label="日程ID", value=event_id, copyable=True - ), - UiHintKeyValuePair( - key="permission", label="权限", value=permission_text - ), - ], - ), - UiHintListBlock( - kind="list", - title="被邀请人", - items=[UiHintListItem(title=email) for email in invited], - emptyText="暂无被邀请人", - ), + items=[ + UiHintKvItem(key="event_id", label="日程ID", value=event_id, copyable=True), + UiHintKvItem(key="permission", label="权限", value=permission_text), ], + list_items=[UiHintListItem(title=email) for email in invited] + if invited + else [], ) diff --git a/backend/src/schemas/agent/__init__.py b/backend/src/schemas/agent/__init__.py index 485d6d0..85e19f4 100644 --- a/backend/src/schemas/agent/__init__.py +++ b/backend/src/schemas/agent/__init__.py @@ -14,7 +14,9 @@ from schemas.agent.runtime_models import ( from schemas.agent.system_agent import AgentType, SystemAgentLLMConfig from schemas.agent.ui_hints import ( UiHintAction, - UiHintBlock, + UiHintIntent, + UiHintSection, + UiHintStatus, UiHintsPayload, ) @@ -29,7 +31,9 @@ __all__ = [ "ToolStatus", "UiMode", "UiHintAction", - "UiHintBlock", + "UiHintIntent", + "UiHintSection", + "UiHintStatus", "UiHintsPayload", "WorkerAgentOutputLite", "WorkerAgentOutputRich", diff --git a/backend/src/schemas/agent/ui_hints.py b/backend/src/schemas/agent/ui_hints.py index 72af622..5d8f26c 100644 --- a/backend/src/schemas/agent/ui_hints.py +++ b/backend/src/schemas/agent/ui_hints.py @@ -1,10 +1,26 @@ +""" +UiHints - 描述性 UI 提示 + +设计原则: +- 描述性而非渲染性: 告诉编译器“要展示什么”，而不是“如何渲染” +- 最小化 token: 保持字段简洁 +- 可编译: 可机械转换为 UiSchemaRenderer +- 尽量无损: hints 中的主要内容字段应尽量被保留到 renderer 中 + +Version: 2.1 +""" + from __future__ import annotations from enum import Enum -from typing import Annotated, Any, Literal +from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field +# ============================================================ +# Enums +# ============================================================ + class UiHintStatus(str, Enum): INFO = "info" @@ -14,6 +30,17 @@ class UiHintStatus(str, Enum): PENDING = "pending" +class UiHintIntent(str, Enum): + """主要展示意图（弱提示，不应决定字段生死）""" + + MESSAGE = "message" # 普通消息/说明 + DATA = "data" # 数据/结果摘要 + LIST = "list" # 列表为主 + STATUS = "status" # 状态结果为主 + FORM = "form" # 结构化内容（当前不表示真实输入表单） + MIXED = "mixed" # 混合内容 + + class UiHintActionStyle(str, Enum): PRIMARY = "primary" SECONDARY = "secondary" @@ -26,520 +53,238 @@ class UiHintTextFormat(str, Enum): MARKDOWN = "markdown" -class UiHintContainerDirection(str, Enum): - VERTICAL = "vertical" - HORIZONTAL = "horizontal" +class UiHintActionType(str, Enum): + NAVIGATION = "navigation" + URL = "url" + EVENT = "event" + TOOL = "tool" + COPY = "copy" + PAYLOAD = "payload" -class UiHintKvLayout(str, Enum): - VERTICAL = "vertical" - HORIZONTAL = "horizontal" - GRID = "grid" +class UiHintIconSource(str, Enum): + ICON = "icon" + EMOJI = "emoji" + URL = "url" -class UiHintOperationType(str, Enum): - CREATE = "create" - UPDATE = "update" - DELETE = "delete" - EXECUTE = "execute" +# ============================================================ +# Base Config +# ============================================================ -class UiHintOperationResult(str, Enum): - SUCCESS = "success" - FAILURE = "failure" - PARTIAL = "partial" - - -class UiHintConfirm(BaseModel): - model_config = ConfigDict(extra="forbid") - - title: str | None = Field( - default=None, - description="Optional confirmation dialog title.", - ) - message: str | None = Field( - default=None, - description="Optional confirmation message shown before action execution.", - ) - confirm_label: str | None = Field( - default=None, - alias="confirmLabel", - description="Optional confirm button label, e.g. 'Delete'.", - ) - cancel_label: str | None = Field( - default=None, - alias="cancelLabel", - description="Optional cancel button label, e.g. 'Cancel'.", +class UiHintBaseModel(BaseModel): + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, ) -class UiHintActionNavigation(BaseModel): - model_config = ConfigDict(extra="forbid") +# ============================================================ +# Action Targets +# ============================================================ + +class UiHintActionNavigation(UiHintBaseModel): type: Literal["navigation"] - path: str = Field( - ..., - description="Internal route path to navigate to.", - ) - params: dict[str, Any] | None = Field( - default=None, - description="Optional route params for internal navigation.", - ) + path: str = Field(..., description="Internal route path.") + params: dict[str, Any] | None = Field(default=None, description="Route params.") -class UiHintActionUrl(BaseModel): - model_config = ConfigDict(extra="forbid") - +class UiHintActionUrl(UiHintBaseModel): type: Literal["url"] - url: str = Field(..., description="External URL to open.") - target: Literal["_self", "_blank"] | None = Field( - default=None, - description="Optional browser target for URL action.", - ) + url: str = Field(..., description="External URL.") + target: Literal["_self", "_blank"] | None = Field(default=None) -class UiHintActionEvent(BaseModel): - model_config = ConfigDict(extra="forbid") - +class UiHintActionEvent(UiHintBaseModel): type: Literal["event"] - event: str = Field( - ..., - description="Frontend domain event name, e.g. 'chat.retry'.", - ) - payload: dict[str, Any] | None = Field( - default=None, - description="Optional event payload for frontend event handling.", - ) + event: str = Field(..., description="Frontend event name.") + payload: dict[str, Any] | None = Field(default=None) -class UiHintActionTool(BaseModel): - model_config = ConfigDict(extra="forbid") - +class UiHintActionTool(UiHintBaseModel): type: Literal["tool"] - tool_id: str = Field( - alias="toolId", - description="Tool identifier used to trigger another tool execution.", - ) - params: dict[str, Any] | None = Field( - default=None, - description="Optional parameters for tool re-execution.", - ) + tool_id: str = Field(alias="toolId", description="Tool identifier.") + params: dict[str, Any] | None = Field(default=None) -class UiHintActionCopy(BaseModel): - model_config = ConfigDict(extra="forbid") - +class UiHintActionCopy(UiHintBaseModel): type: Literal["copy"] - content: str = Field(..., description="Text content to copy to clipboard.") - success_message: str | None = Field( - default=None, - alias="successMessage", - description="Optional user-facing success message after copy.", - ) + content: str = Field(..., description="Content to copy.") + success_message: str | None = Field(alias="successMessage", default=None) -class UiHintActionPayload(BaseModel): - model_config = ConfigDict(extra="forbid") - +class UiHintActionPayload(UiHintBaseModel): type: Literal["payload"] - payload: dict[str, Any] = Field( - ..., - description="Structured payload to submit to frontend or gateway.", - ) - submit_to: str | None = Field( - default=None, - alias="submitTo", - description="Optional submit target path or endpoint key.", - ) + payload: dict[str, Any] = Field(..., description="Structured payload.") + submit_to: str | None = Field(alias="submitTo", default=None) -UiHintActionTarget = Annotated[ - ( - UiHintActionNavigation - | UiHintActionUrl - | UiHintActionEvent - | UiHintActionTool - | UiHintActionCopy - | UiHintActionPayload - ), - Field(discriminator="type"), -] +UiHintActionTarget = ( + UiHintActionNavigation + | UiHintActionUrl + | UiHintActionEvent + | UiHintActionTool + | UiHintActionCopy + | UiHintActionPayload +) -class UiHintAction(BaseModel): - model_config = ConfigDict( - extra="forbid", - json_schema_extra={ - "examples": [ - { - "id": "action-open-calendar", - "label": "Open calendar", - "style": "primary", - "action": {"type": "navigation", "path": "/calendar"}, - } - ] - }, - ) - - id: str | None = Field( - default=None, - description="Optional stable action id for tracking and targeting.", - ) - label: str = Field( - ..., - description="User-facing action label shown on button/link.", - ) - style: UiHintActionStyle | None = Field( - default=None, - description="Optional semantic button style.", - ) - disabled: bool = Field( - default=False, - description="Whether this action should be rendered as disabled.", - ) - action: UiHintActionTarget = Field( - ..., - description="Executable action target definition.", - ) - confirm: UiHintConfirm | None = Field( - default=None, - description="Optional confirmation requirement before execution.", - ) +class UiHintAction(UiHintBaseModel): + label: str = Field(..., description="Button label.") + style: UiHintActionStyle | None = Field(default=None, description="Button style.") + disabled: bool = Field(default=False, description="Disabled state.") + action: UiHintActionTarget = Field(..., description="Action to execute.") -class UiHintIcon(BaseModel): - model_config = ConfigDict(extra="forbid") - - source: Literal["icon", "emoji", "url"] = Field( - ..., - description="Icon source type.", - ) - value: str = Field( - ..., - description="Icon identifier, emoji text, or image URL based on source.", - ) - color: str | None = Field( - default=None, - description="Optional semantic color hint. Do not encode pixel-level style rules.", - ) - size: int | None = Field( - default=None, - description="Optional icon size hint in abstract UI units.", - ) +# ============================================================ +# Small Descriptive Models +# ============================================================ -class UiHintBadge(BaseModel): - model_config = ConfigDict(extra="forbid") - - label: str = Field(..., description="Badge text label.") - variant: Literal["default", "success", "warning", "error", "info"] = Field( - default="default", - description="Semantic badge variant.", - ) +class UiHintIcon(UiHintBaseModel): + source: UiHintIconSource = Field(default=UiHintIconSource.ICON) + value: str = Field(..., description="Icon identifier / emoji / url.") + color: str | None = Field(default=None) + size: int | None = Field(default=None) -class UiHintKeyValuePair(BaseModel): - model_config = ConfigDict(extra="forbid") - - key: str = Field(..., description="Stable key identifier for this pair.") - label: str | None = Field( - default=None, - description="Optional user-facing label. Fallback to key when missing.", - ) - value: str | int | bool | None = Field( - default=None, - description="Scalar value for this key-value pair.", - ) - copyable: bool = Field( - default=False, - description="Whether frontend may offer copy interaction for this value.", - ) +class UiHintKvItem(UiHintBaseModel): + key: str = Field(..., description="Key identifier.") + label: str | None = Field(default=None, description="Display label.") + value: Any = Field(default=None, description="Value.") + copyable: bool = Field(default=False, description="Allow copy.") -class UiHintListItem(BaseModel): - model_config = ConfigDict(extra="forbid") - - id: str | None = Field( - default=None, - description="Optional stable list item id.", - ) - title: str = Field(..., description="Primary list item title.") - subtitle: str | None = Field( - default=None, - description="Optional short secondary text.", - ) - description: str | None = Field( - default=None, - description="Optional detailed description for this item.", - ) - icon: UiHintIcon | None = Field( - default=None, - description="Optional semantic icon metadata.", - ) - badge: UiHintBadge | None = Field( - default=None, - description="Optional semantic badge metadata.", - ) - metadata: dict[str, Any] = Field( - default_factory=dict, - description="Optional non-visual metadata for analytics or interactions.", - ) - actions: list[UiHintAction] = Field( - default_factory=list, - description="Optional per-item actions, recommended up to 3.", - ) +class UiHintListItem(UiHintBaseModel): + id: str | None = Field(default=None) + title: str = Field(..., description="Item title.") + subtitle: str | None = Field(default=None) + description: str | None = Field(default=None) + icon: UiHintIcon | None = Field(default=None) + status: UiHintStatus | None = Field(default=None) + actions: list[UiHintAction] = Field(default_factory=list) -class UiHintPagination(BaseModel): - model_config = ConfigDict(extra="forbid") +class UiHintSection(UiHintBaseModel): + title: str | None = Field(default=None, description="Section title.") + description: str | None = Field(default=None, description="Section description.") + icon: UiHintIcon | None = Field(default=None, description="Section icon.") - page: int = Field(..., description="Current page number starting from 1.") - page_size: int = Field( - alias="pageSize", - description="Page size used for this list page.", - ) - total: int = Field(..., description="Total number of records.") - has_more: bool = Field( - alias="hasMore", - description="Whether there are more pages after current page.", - ) - - -class UiHintBaseBlock(BaseModel): - model_config = ConfigDict(extra="forbid") - - id: str | None = Field( - default=None, - description="Optional stable block id.", - ) - title: str | None = Field( - default=None, - description="Optional block title.", - ) - description: str | None = Field( - default=None, - description="Optional block description.", - ) - status: UiHintStatus | None = Field( - default=None, - description="Optional semantic status for this block.", - ) - actions: list[UiHintAction] = Field( - default_factory=list, - description="Optional block-level actions, recommended up to 3.", - ) - - -class UiHintTextBlock(UiHintBaseBlock): - kind: Literal["text"] - content: str = Field( - ..., - description="Main text content to present.", - ) - format: UiHintTextFormat = Field( + content: str | None = Field(default=None, description="Main text content.") + content_format: UiHintTextFormat = Field( default=UiHintTextFormat.PLAIN, - description="Text format: plain or markdown.", + alias="contentFormat", + description="Section content text format.", ) - -class UiHintCardBlock(UiHintBaseBlock): - kind: Literal["card"] - children: list["UiHintBlock"] = Field( + items: list[UiHintKvItem] = Field(default_factory=list, description="KV items.") + list_items: list[UiHintListItem] = Field( default_factory=list, - description="Nested child blocks grouped under this card.", + alias="listItems", + description="List items.", ) + actions: list[UiHintAction] = Field(default_factory=list, description="Actions.") -class UiHintKvBlock(UiHintBaseBlock): - kind: Literal["kv"] - pairs: list[UiHintKeyValuePair] = Field( - default_factory=list, - description="Key-value pairs to display.", - ) - layout: UiHintKvLayout = Field( - default=UiHintKvLayout.VERTICAL, - description="Preferred semantic layout for key-value content.", - ) +# ============================================================ +# Root Payload +# ============================================================ -class UiHintListBlock(UiHintBaseBlock): - kind: Literal["list"] - items: list[UiHintListItem] = Field( - default_factory=list, - description="List items to present.", - ) - pagination: UiHintPagination | None = Field( - default=None, - description="Optional pagination metadata.", - ) - empty_text: str | None = Field( - default=None, - alias="emptyText", - description="Optional message shown when list items are empty.", - ) +class UiHintsPayload(UiHintBaseModel): + """ + 描述性 UI 提示 + 设计目标: + - agent 输出尽可能短 + - 不表达布局细节 + - 编译器负责转换为完整 UiSchemaRenderer + """ -class UiHintOperationBlock(UiHintBaseBlock): - kind: Literal["operation"] - operation: UiHintOperationType = Field( - ..., - description="Operation category: create/update/delete/execute.", - ) - result: UiHintOperationResult = Field( - ..., - description="Operation result: success/failure/partial.", - ) - message: str | None = Field( - default=None, - description="Optional operation summary message.", - ) - affected_count: int | None = Field( - default=None, - alias="affectedCount", - description="Optional affected record count.", - ) - details: dict[str, Any] | None = Field( - default=None, - description="Optional machine-readable operation details.", - ) - - -class UiHintErrorBlock(UiHintBaseBlock): - kind: Literal["error"] - error_code: str = Field( - alias="errorCode", - description="Stable error code for categorization.", - ) - message: str = Field( - ..., - description="Human-readable error message.", - ) - retryable: bool = Field( - default=False, - description="Whether retry is likely to succeed.", - ) - details: str | None = Field( - default=None, - description="Optional plain-text diagnostic details.", - ) - suggestions: list[str] = Field( - default_factory=list, - description="Optional actionable suggestions, recommended up to 3.", - ) - - -class UiHintContainerBlock(UiHintBaseBlock): - kind: Literal["container"] - direction: UiHintContainerDirection = Field( - default=UiHintContainerDirection.VERTICAL, - description="Child block layout direction.", - ) - gap: int | None = Field( - default=None, - description="Optional semantic spacing hint between children.", - ) - children: list["UiHintBlock"] = Field( - default_factory=list, - description="Nested child blocks in this container.", - ) - - -class UiHintCustomBlock(UiHintBaseBlock): - kind: Literal["custom"] - renderer_key: str = Field( - alias="rendererKey", - description=( - "Custom semantic renderer key. Use only when standard block kinds " - "cannot represent the intent." - ), - ) - payload: dict[str, Any] = Field( - default_factory=dict, - description="Structured custom payload consumed by the renderer.", - ) - - -UiHintBlock = Annotated[ - ( - UiHintTextBlock - | UiHintCardBlock - | UiHintKvBlock - | UiHintListBlock - | UiHintOperationBlock - | UiHintErrorBlock - | UiHintContainerBlock - | UiHintCustomBlock - ), - Field(discriminator="kind"), -] - - -class UiHintsPayload(BaseModel): model_config = ConfigDict( extra="forbid", + populate_by_name=True, json_schema_extra={ "examples": [ { - "version": "1.0", - "status": "info", - "title": "Schedule update", - "blocks": [ - { - "kind": "text", - "content": "Your meeting is moved to 3:00 PM.", - "format": "plain", - }, - { - "kind": "list", - "title": "Next steps", - "items": [ - {"title": "Open calendar"}, - {"title": "Notify attendees"}, - ], - }, + "intent": "status", + "status": "success", + "title": "日程已创建", + "body": "本次创建已成功完成。", + "items": [ + {"key": "title", "label": "主题", "value": "Q1 规划会议"}, + {"key": "time", "label": "时间", "value": "2026-03-15 14:00"}, ], "actions": [ { - "label": "Open calendar", + "label": "查看详情", "style": "primary", - "action": {"type": "navigation", "path": "/calendar"}, - } + "action": { + "type": "navigation", + "path": "/calendar/evt_123", + }, + }, + { + "label": "删除", + "style": "danger", + "action": { + "type": "tool", + "toolId": "calendar.delete", + "params": {"eventId": "evt_123"}, + }, + }, ], - "meta": {"source": "worker"}, } ] }, ) - version: str = Field( - default="1.0", - description="Ui hints payload version.", + version: str = Field(default="2.1") + + intent: UiHintIntent = Field( + default=UiHintIntent.MESSAGE, + description="Primary display intent.", ) status: UiHintStatus = Field( default=UiHintStatus.INFO, - description="Overall semantic status for the full ui_hints payload.", + description="Overall status.", ) - title: str | None = Field( - default=None, - description="Optional top-level semantic title.", + + title: str | None = Field(default=None, description="Top-level title.") + description: str | None = Field(default=None, description="Top-level description.") + + body: str | None = Field(default=None, description="Top-level main body text.") + body_format: UiHintTextFormat = Field( + default=UiHintTextFormat.PLAIN, + alias="bodyFormat", + description="Body text format.", ) - description: str | None = Field( - default=None, - description="Optional top-level semantic description.", - ) - blocks: list[UiHintBlock] = Field( + + items: list[UiHintKvItem] = Field( default_factory=list, - description="Main semantic content blocks.", + description="Top-level key-value items.", + ) + list_items: list[UiHintListItem] = Field( + default_factory=list, + alias="listItems", + description="Top-level list items.", + ) + sections: list[UiHintSection] = Field( + default_factory=list, + description="Grouped sections.", ) actions: list[UiHintAction] = Field( default_factory=list, - description="Optional top-level actions, recommended up to 3.", + description="Top-level actions.", + ) + + icon: UiHintIcon | None = Field( + default=None, + description="Top-level icon.", ) meta: dict[str, Any] = Field( default_factory=dict, - description="Optional non-visual metadata for tracing and integration.", + description="Extra meta, e.g. requestId/toolId/traceId/userId.", ) - - -UiHintCardBlock.model_rebuild() -UiHintContainerBlock.model_rebuild() diff --git a/backend/src/schemas/agent/ui_schema.py b/backend/src/schemas/agent/ui_schema.py index f35fea6..11104cb 100644 --- a/backend/src/schemas/agent/ui_schema.py +++ b/backend/src/schemas/agent/ui_schema.py @@ -1,10 +1,12 @@ """ -UI Schema Protocol Implementation. +UI Schema Renderer Protocol -This module is the single source of truth for UI Schema. -All implementations must follow docs/protocols/ui-schema.md. +目标： +- 只保留“基础组件 + 布局容器” +- 最终返回一个 UiSchemaRenderer +- 前端只需要递归渲染 root 布局树即可 -Version: 1.0 +Version: 2.0 """ from __future__ import annotations @@ -12,21 +14,12 @@ from __future__ import annotations from enum import Enum from typing import Any, Literal, NotRequired, TypedDict, Union - -# ========== Enums ========== - - -class SchemaType(str, Enum): - """Schema type identifier.""" - - TOOL_RESULT = "tool_result" - AGENT_RESPONSE = "agent_response" - NOTIFICATION = "notification" +# ============================================================ +# Enums +# ============================================================ class UiStatus(str, Enum): - """Unified status for all nodes.""" - INFO = "info" SUCCESS = "success" WARNING = "warning" @@ -35,736 +28,600 @@ class UiStatus(str, Enum): class IconSource(str, Enum): - """Icon source type.""" - ICON = "icon" EMOJI = "emoji" URL = "url" -class ActionType(str, Enum): - """Action type identifier.""" - - NAVIGATION = "navigation" - URL = "url" - EVENT = "event" - TOOL = "tool" - COPY = "copy" - PAYLOAD = "payload" - - -class OperationType(str, Enum): - """Operation node operation type.""" - - CREATE = "create" - UPDATE = "update" - DELETE = "delete" - EXECUTE = "execute" - - -class OperationResult(str, Enum): - """Operation node result type.""" - - SUCCESS = "success" - FAILURE = "failure" - PARTIAL = "partial" - - -class ContainerDirection(str, Enum): - """Container node direction.""" - - VERTICAL = "vertical" - HORIZONTAL = "horizontal" - - class TextFormat(str, Enum): - """Text node format.""" - PLAIN = "plain" MARKDOWN = "markdown" -class KvLayout(str, Enum): - """Key-value node layout.""" - - VERTICAL = "vertical" - HORIZONTAL = "horizontal" - GRID = "grid" +class TextRole(str, Enum): + TITLE = "title" + SUBTITLE = "subtitle" + BODY = "body" + CAPTION = "caption" + CODE = "code" -class BadgeVariant(str, Enum): - """Badge variant.""" - - DEFAULT = "default" - SUCCESS = "success" - WARNING = "warning" - ERROR = "error" - INFO = "info" - - -class ActionStyle(str, Enum): - """Action button style.""" - +class ButtonStyle(str, Enum): PRIMARY = "primary" SECONDARY = "secondary" GHOST = "ghost" DANGER = "danger" +class LayoutDirection(str, Enum): + VERTICAL = "vertical" + HORIZONTAL = "horizontal" + + +class LayoutAppearance(str, Enum): + PLAIN = "plain" + CARD = "card" + SECTION = "section" + + +class LayoutAlign(str, Enum): + START = "start" + CENTER = "center" + END = "end" + STRETCH = "stretch" + + +class LayoutJustify(str, Enum): + START = "start" + CENTER = "center" + END = "end" + SPACE_BETWEEN = "space-between" + + class RendererTheme(str, Enum): - """Renderer theme.""" - DEFAULT = "default" - DARK = "dark" LIGHT = "light" + DARK = "dark" -# ========== Common Types ========== +# ============================================================ +# Meta +# ============================================================ -class UiIcon(TypedDict, total=False): - """Icon structure.""" +class UiMeta(TypedDict, total=False): + requestId: str + toolId: str + traceId: str + userId: str - source: IconSource + +# ============================================================ +# Action Payloads +# ============================================================ + + +class NavigateAction(TypedDict, total=False): + type: Literal["navigation"] + path: str + params: dict[str, Any] + + +class UrlAction(TypedDict, total=False): + type: Literal["url"] + url: str + target: Literal["_self", "_blank"] + + +class EventAction(TypedDict, total=False): + type: Literal["event"] + event: str + payload: dict[str, Any] + + +class ToolAction(TypedDict, total=False): + type: Literal["tool"] + toolId: str + params: dict[str, Any] + + +class CopyAction(TypedDict, total=False): + type: Literal["copy"] + content: str + successMessage: str + + +class PayloadAction(TypedDict, total=False): + type: Literal["payload"] + payload: dict[str, Any] + submitTo: str + + +UiActionPayload = Union[ + NavigateAction, + UrlAction, + EventAction, + ToolAction, + CopyAction, + PayloadAction, +] + + +# ============================================================ +# Shared Small Types +# ============================================================ + + +class UiIconSpec(TypedDict, total=False): + source: str value: str color: str size: int -class UiBadge(TypedDict, total=False): - """Badge structure.""" - - label: str - variant: BadgeVariant - - -class Pagination(TypedDict): - """Pagination info.""" - - page: int - pageSize: int - total: int - hasMore: bool - - -class ActionConfirm(TypedDict, total=False): - """Action confirmation config.""" - - title: str - message: str - confirmLabel: str - cancelLabel: str - - -class KeyValuePair(TypedDict, total=False): - """Key-value pair.""" - +class UiKvItem(TypedDict, total=False): key: str label: str - value: Union[str, int, bool] + value: Any copyable: bool -class TableColumn(TypedDict, total=False): - """Table column definition.""" - - key: str - label: str - width: str - align: Literal["left", "center", "right"] - - -class TableRow(TypedDict, total=False): - """Table row.""" - +class UiBaseNode(TypedDict, total=False): id: str - cells: dict[str, Any] - metadata: dict[str, Any] - actions: list[dict[str, Any]] + visible: bool -class ListItem(TypedDict, total=False): - """List item.""" - - id: str - title: str - subtitle: str - description: str - icon: UiIcon - badge: UiBadge - metadata: dict[str, Any] - actions: list[dict[str, Any]] +# ============================================================ +# Primitive Components +# ============================================================ -# ========== Action Types ========== - - -class NavigateAction(TypedDict): - """Navigate to internal path.""" - - type: Literal["navigation"] - path: str - params: NotRequired[dict[str, Any]] - - -class LinkAction(TypedDict): - """Open external URL.""" - - type: Literal["url"] - url: str - target: NotRequired[Literal["_self", "_blank"]] - - -class EventAction(TypedDict): - """Trigger frontend event.""" - - type: Literal["event"] - event: str - payload: NotRequired[dict[str, Any]] - - -class ToolAction(TypedDict): - """Re-execute a tool.""" - - type: Literal["tool"] - toolId: str - params: NotRequired[dict[str, Any]] - - -class CopyAction(TypedDict): - """Copy content to clipboard.""" - - type: Literal["copy"] +class UiTextNode(UiBaseNode, total=False): + type: Literal["text"] content: str - successMessage: NotRequired[str] + format: str # TextFormat + role: str # TextRole + status: str # UiStatus + maxLines: int -class PayloadAction(TypedDict): - """Submit payload to endpoint.""" - - type: Literal["payload"] - payload: dict[str, Any] - submitTo: NotRequired[str] +class UiIconNode(UiBaseNode, total=False): + type: Literal["icon"] + source: str # IconSource + value: str + color: str + size: int -# ========== Action ========== - - -class UiAction(TypedDict, total=False): - """Action structure.""" - - id: str +class UiBadgeNode(UiBaseNode, total=False): + type: Literal["badge"] label: str - icon: UiIcon - style: ActionStyle + status: str # UiStatus + + +class UiButtonNode(UiBaseNode, total=False): + type: Literal["button"] + label: str + style: str # ButtonStyle disabled: bool - action: ( - NavigateAction - | LinkAction - | EventAction - | ToolAction - | CopyAction - | PayloadAction - ) - confirm: ActionConfirm + icon: UiIconSpec + action: UiActionPayload -# ========== Node Types (using dict for simplicity) ========== - -# Type alias for any node -UiNode = dict[str, Any] +class UiKvNode(UiBaseNode, total=False): + type: Literal["kv"] + items: list[UiKvItem] + columns: int -# ========== Builder Functions ========== +class UiDividerNode(UiBaseNode, total=False): + type: Literal["divider"] + inset: int -def build_document( - status: UiStatus, - nodes: list[UiNode], +# ============================================================ +# Layout Containers +# ============================================================ + + +class UiStackNode(UiBaseNode, total=False): + type: Literal["stack"] + direction: str # LayoutDirection + gap: int + appearance: str # LayoutAppearance + status: str # UiStatus + align: str # LayoutAlign + justify: str # LayoutJustify + wrap: bool + children: list["UiNode"] + + +class UiGridNode(UiBaseNode, total=False): + type: Literal["grid"] + columns: int + gap: int + appearance: str # LayoutAppearance + status: str # UiStatus + children: list["UiNode"] + + +UiNode = Union[ + UiTextNode, + UiIconNode, + UiBadgeNode, + UiButtonNode, + UiKvNode, + UiDividerNode, + UiStackNode, + UiGridNode, +] + +UiLayoutNode = Union[UiStackNode, UiGridNode] + + +# ============================================================ +# Root Renderer +# ============================================================ + + +class UiSchemaRenderer(TypedDict, total=False): + version: str + locale: str + status: str # UiStatus + theme: str # RendererTheme + meta: UiMeta + root: UiLayoutNode + + +# ============================================================ +# Root Builder +# ============================================================ + + +def build_renderer( + root: UiLayoutNode, *, - version: str = "1.0", - schema_type: SchemaType = SchemaType.TOOL_RESULT, - doc_id: str | None = None, - timestamp: str | None = None, + version: str = "2.0", locale: str = "zh-CN", - renderer: dict[str, Any] | None = None, - meta: dict[str, Any] | None = None, -) -> dict[str, Any]: - """Build a UI schema document.""" - doc: dict[str, Any] = { + status: UiStatus = UiStatus.INFO, + theme: RendererTheme = RendererTheme.DEFAULT, + meta: UiMeta | None = None, +) -> UiSchemaRenderer: + renderer: UiSchemaRenderer = { "version": version, - "schemaType": schema_type.value, + "locale": locale, "status": status.value, - "nodes": nodes, + "theme": theme.value, + "root": root, } - if doc_id: - doc["docId"] = doc_id - if timestamp: - doc["timestamp"] = timestamp - if locale: - doc["locale"] = locale - if renderer: - doc["renderer"] = renderer if meta: - doc["meta"] = meta - return doc + renderer["meta"] = meta + return renderer -def build_success_document( - nodes: list[UiNode], - **kwargs: Any, -) -> dict[str, Any]: - """Build a success document.""" - return build_document(status=UiStatus.SUCCESS, nodes=nodes, **kwargs) +# ============================================================ +# Primitive Builders +# ============================================================ -def build_error_document( - nodes: list[UiNode], - **kwargs: Any, -) -> dict[str, Any]: - """Build an error document.""" - return build_document(status=UiStatus.ERROR, nodes=nodes, **kwargs) - - -def build_text_node( +def build_text( content: str, *, node_id: str | None = None, format: TextFormat = TextFormat.PLAIN, - icon: dict[str, Any] | None = None, - actions: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - """Build a text node.""" - node: dict[str, Any] = { + role: TextRole = TextRole.BODY, + status: UiStatus | None = None, + max_lines: int | None = None, + visible: bool = True, +) -> UiTextNode: + node: UiTextNode = { "type": "text", "content": content, "format": format.value, + "role": role.value, + "visible": visible, } if node_id: node["id"] = node_id - if icon: - node["icon"] = icon - if actions: - node["actions"] = actions - return node - - -def build_card_node( - *, - node_id: str | None = None, - title: str | None = None, - description: str | None = None, - icon: dict[str, Any] | None = None, - status: UiStatus | None = None, - timestamp: str | None = None, - children: list[UiNode] | None = None, - footer: dict[str, Any] | None = None, - extensions: dict[str, Any] | None = None, - actions: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - """Build a card node.""" - node: dict[str, Any] = {"type": "card"} - if node_id: - node["id"] = node_id - if title: - node["title"] = title - if description: - node["description"] = description - if icon: - node["icon"] = icon if status: node["status"] = status.value - if timestamp: - node["timestamp"] = timestamp - if children: - node["children"] = children - if footer: - node["footer"] = footer - if extensions: - node["extensions"] = extensions - if actions: - node["actions"] = actions + if max_lines is not None: + node["maxLines"] = max_lines return node -def build_kv_node( - pairs: list[dict[str, Any]], - *, - node_id: str | None = None, - title: str | None = None, - description: str | None = None, - icon: dict[str, Any] | None = None, - status: UiStatus | None = None, - layout: KvLayout = KvLayout.VERTICAL, - extensions: dict[str, Any] | None = None, - actions: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - """Build a key-value node.""" - node: dict[str, Any] = { - "type": "kv", - "pairs": pairs, - "layout": layout.value, - } - if node_id: - node["id"] = node_id - if title: - node["title"] = title - if description: - node["description"] = description - if icon: - node["icon"] = icon - if status: - node["status"] = status.value - if extensions: - node["extensions"] = extensions - if actions: - node["actions"] = actions - return node - - -def build_list_node( - items: list[dict[str, Any]], - *, - node_id: str | None = None, - title: str | None = None, - description: str | None = None, - icon: dict[str, Any] | None = None, - status: UiStatus | None = None, - pagination: dict[str, Any] | None = None, - empty_text: str | None = None, - extensions: dict[str, Any] | None = None, - actions: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - """Build a list node.""" - node: dict[str, Any] = { - "type": "list", - "items": items, - } - if node_id: - node["id"] = node_id - if title: - node["title"] = title - if description: - node["description"] = description - if icon: - node["icon"] = icon - if status: - node["status"] = status.value - if pagination: - node["pagination"] = pagination - if empty_text: - node["emptyText"] = empty_text - if extensions: - node["extensions"] = extensions - if actions: - node["actions"] = actions - return node - - -def build_error_node( - error_code: str, - message: str, - *, - node_id: str | None = None, - title: str | None = None, - icon: dict[str, Any] | None = None, - details: str | None = None, - stack: str | None = None, - retryable: bool = False, - suggestions: list[str] | None = None, - retry: dict[str, Any] | None = None, - support: dict[str, Any] | None = None, - actions: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - """Build an error node.""" - node: dict[str, Any] = { - "type": "error", - "errorCode": error_code, - "message": message, - "retryable": retryable, - } - if node_id: - node["id"] = node_id - if title: - node["title"] = title - if icon: - node["icon"] = icon - if details: - node["details"] = details - if stack: - node["stack"] = stack - if suggestions: - node["suggestions"] = suggestions - if retry: - node["retry"] = retry - if support: - node["support"] = support - if actions: - node["actions"] = actions - return node - - -def build_operation_node( - operation: OperationType, - result: OperationResult, - *, - node_id: str | None = None, - title: str | None = None, - description: str | None = None, - icon: dict[str, Any] | None = None, - status: UiStatus | None = None, - message: str | None = None, - affected_count: int | None = None, - details: dict[str, Any] | None = None, - rollback: dict[str, Any] | None = None, - extensions: dict[str, Any] | None = None, - actions: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - """Build an operation node.""" - node: dict[str, Any] = { - "type": "operation", - "operation": operation.value, - "result": result.value, - } - if node_id: - node["id"] = node_id - if title: - node["title"] = title - if description: - node["description"] = description - if icon: - node["icon"] = icon - if status: - node["status"] = status.value - if message: - node["message"] = message - if affected_count is not None: - node["affectedCount"] = affected_count - if details: - node["details"] = details - if rollback: - node["rollback"] = rollback - if extensions: - node["extensions"] = extensions - if actions: - node["actions"] = actions - return node - - -def build_container_node( - children: list[UiNode], - direction: ContainerDirection = ContainerDirection.VERTICAL, - *, - node_id: str | None = None, - gap: int | None = None, - actions: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - """Build a container node.""" - node: dict[str, Any] = { - "type": "container", - "direction": direction.value, - "children": children, - } - if node_id: - node["id"] = node_id - if gap is not None: - node["gap"] = gap - if actions: - node["actions"] = actions - return node - - -def build_action( - action_id: str, - label: str, - action: ( - NavigateAction - | LinkAction - | EventAction - | ToolAction - | CopyAction - | PayloadAction - ), - *, - icon: dict[str, Any] | None = None, - style: ActionStyle | None = None, - disabled: bool = False, - confirm: dict[str, Any] | None = None, -) -> dict[str, Any]: - """Build an action.""" - act: dict[str, Any] = { - "id": action_id, - "label": label, - "action": action, - } - if icon: - act["icon"] = icon - if style: - act["style"] = style.value - if disabled: - act["disabled"] = disabled - if confirm: - act["confirm"] = confirm - return act - - def build_icon( source: IconSource, value: str, *, + node_id: str | None = None, color: str | None = None, size: int | None = None, -) -> dict[str, Any]: - """Build an icon.""" - icon: dict[str, Any] = {"source": source.value, "value": value} + visible: bool = True, +) -> UiIconNode: + node: UiIconNode = { + "type": "icon", + "source": source.value, + "value": value, + "visible": visible, + } + if node_id: + node["id"] = node_id if color: - icon["color"] = color + node["color"] = color if size is not None: - icon["size"] = size - return icon + node["size"] = size + return node -# ========== Legacy Compatibility Wrappers ========== -# These wrappers maintain API compatibility with existing code +def build_badge( + label: str, + *, + node_id: str | None = None, + status: UiStatus = UiStatus.INFO, + visible: bool = True, +) -> UiBadgeNode: + node: UiBadgeNode = { + "type": "badge", + "label": label, + "status": status.value, + "visible": visible, + } + if node_id: + node["id"] = node_id + return node + + +def build_button( + label: str, + action: UiActionPayload, + *, + node_id: str | None = None, + style: ButtonStyle = ButtonStyle.PRIMARY, + disabled: bool = False, + icon: UiIconSpec | None = None, + visible: bool = True, +) -> UiButtonNode: + node: UiButtonNode = { + "type": "button", + "label": label, + "style": style.value, + "disabled": disabled, + "action": action, + "visible": visible, + } + if node_id: + node["id"] = node_id + if icon: + node["icon"] = icon + return node + + +def build_kv( + items: list[UiKvItem], + *, + node_id: str | None = None, + columns: int = 1, + visible: bool = True, +) -> UiKvNode: + node: UiKvNode = { + "type": "kv", + "items": items, + "columns": columns, + "visible": visible, + } + if node_id: + node["id"] = node_id + return node + + +def build_divider( + *, + node_id: str | None = None, + inset: int = 0, + visible: bool = True, +) -> UiDividerNode: + node: UiDividerNode = { + "type": "divider", + "inset": inset, + "visible": visible, + } + if node_id: + node["id"] = node_id + return node + + +# ============================================================ +# Layout Builders +# ============================================================ + + +def build_stack( + children: list[UiNode], + *, + node_id: str | None = None, + direction: LayoutDirection = LayoutDirection.VERTICAL, + gap: int = 12, + appearance: LayoutAppearance = LayoutAppearance.PLAIN, + status: UiStatus | None = None, + align: LayoutAlign = LayoutAlign.START, + justify: LayoutJustify = LayoutJustify.START, + wrap: bool = False, + visible: bool = True, +) -> UiStackNode: + node: UiStackNode = { + "type": "stack", + "direction": direction.value, + "gap": gap, + "appearance": appearance.value, + "align": align.value, + "justify": justify.value, + "wrap": wrap, + "children": children, + "visible": visible, + } + if node_id: + node["id"] = node_id + if status: + node["status"] = status.value + return node + + +def build_grid( + children: list[UiNode], + *, + columns: int, + node_id: str | None = None, + gap: int = 12, + appearance: LayoutAppearance = LayoutAppearance.PLAIN, + status: UiStatus | None = None, + visible: bool = True, +) -> UiGridNode: + node: UiGridNode = { + "type": "grid", + "columns": columns, + "gap": gap, + "appearance": appearance.value, + "children": children, + "visible": visible, + } + if node_id: + node["id"] = node_id + if status: + node["status"] = status.value + return node + + +# ============================================================ +# Small Action Builders +# ============================================================ + + +def action_navigation( + path: str, params: dict[str, Any] | None = None +) -> NavigateAction: + action: NavigateAction = {"type": "navigation", "path": path} + if params: + action["params"] = params + return action + + +def action_url(url: str, target: Literal["_self", "_blank"] = "_blank") -> UrlAction: + return {"type": "url", "url": url, "target": target} + + +def action_event(event: str, payload: dict[str, Any] | None = None) -> EventAction: + action: EventAction = {"type": "event", "event": event} + if payload: + action["payload"] = payload + return action + + +def action_tool(tool_id: str, params: dict[str, Any] | None = None) -> ToolAction: + action: ToolAction = {"type": "tool", "toolId": tool_id} + if params: + action["params"] = params + return action + + +def action_copy(content: str, success_message: str | None = None) -> CopyAction: + action: CopyAction = {"type": "copy", "content": content} + if success_message: + action["successMessage"] = success_message + return action + + +def action_payload( + payload: dict[str, Any], submit_to: str | None = None +) -> PayloadAction: + action: PayloadAction = {"type": "payload", "payload": payload} + if submit_to: + action["submitTo"] = submit_to + return action + + +# ============================================================ +# Derived Helpers (协议外的便捷封装，不是基础原语) +# ============================================================ def build_card( - card_type: str, - data: dict[str, Any], + children: list[UiNode], *, - version: str = "1.0", - actions: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - """Legacy wrapper - builds a card node.""" - return { - "type": card_type, - "version": version, - "data": data, - "actions": actions or [], - } + node_id: str | None = None, + gap: int = 12, + status: UiStatus | None = None, +) -> UiStackNode: + return build_stack( + children, + node_id=node_id, + direction=LayoutDirection.VERTICAL, + gap=gap, + appearance=LayoutAppearance.CARD, + status=status, + ) -def build_calendar_card( +def build_section( title: str, - start_at: str, + children: list[UiNode], *, - id: str | None = None, - end_at: str | None = None, + node_id: str | None = None, description: str | None = None, - timezone: str | None = None, - location: str | None = None, - color: str | None = None, - source_type: str | None = None, - actions: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - """Legacy wrapper for calendar card.""" - data: dict[str, Any] = { - "title": title, - "startAt": start_at, - } - if id is not None: - data["id"] = id - if end_at is not None: - data["endAt"] = end_at - if description is not None: - data["description"] = description - if timezone is not None: - data["timezone"] = timezone - if location is not None: - data["location"] = location - if color is not None: - data["color"] = color - if source_type is not None: - data["sourceType"] = source_type + status: UiStatus | None = None, + gap: int = 12, +) -> UiStackNode: + header_nodes: list[UiNode] = [build_text(title, role=TextRole.TITLE)] + if description: + header_nodes.append(build_text(description, role=TextRole.CAPTION)) - return build_card( - card_type="calendar_card.v1", - data=data, - actions=actions, + all_children = header_nodes + children + return build_stack( + all_children, + node_id=node_id, + direction=LayoutDirection.VERTICAL, + gap=gap, + appearance=LayoutAppearance.SECTION, + status=status, ) -def build_calendar_list( - items: list[dict[str, Any]], - *, - page: int = 1, - page_size: int = 20, - total: int | None = None, - total_pages: int | None = None, -) -> dict[str, Any]: - """Legacy wrapper for calendar list.""" - pagination: dict[str, Any] = { - "page": page, - "pageSize": page_size, - } - if total is not None: - pagination["total"] = total - if total_pages is not None: - pagination["totalPages"] = total_pages - - return build_card( - card_type="calendar_event_list.v1", - data={ - "items": items, - "pagination": pagination, - }, - ) - - -def build_calendar_operation( - operation: str, - ok: bool, +def build_status_panel( + title: str, message: str, *, - code: str | None = None, - actions: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - """Legacy wrapper for calendar operation.""" - data: dict[str, Any] = { - "operation": operation, - "ok": ok, - "message": message, - } - if code is not None: - data["code"] = code + status: UiStatus, + primary_button: UiButtonNode | None = None, + secondary_button: UiButtonNode | None = None, + node_id: str | None = None, +) -> UiStackNode: + children: list[UiNode] = [ + build_stack( + [ + build_text(title, role=TextRole.TITLE), + build_badge(label=status.value.upper(), status=status), + ], + direction=LayoutDirection.HORIZONTAL, + gap=8, + align=LayoutAlign.CENTER, + justify=LayoutJustify.SPACE_BETWEEN, + ), + build_text(message, role=TextRole.BODY, status=status), + ] - return build_card( - card_type="calendar_operation.v1", - data=data, - actions=actions, - ) + actions: list[UiNode] = [] + if primary_button: + actions.append(primary_button) + if secondary_button: + actions.append(secondary_button) + if actions: + children.append( + build_stack( + actions, + direction=LayoutDirection.HORIZONTAL, + gap=8, + ) + ) -def build_error_card( - message: str, - *, - code: str | None = None, -) -> dict[str, Any]: - """Legacy wrapper for error card.""" - data: dict[str, Any] = {"message": message} - if code is not None: - data["code"] = code - - return build_card( - card_type="error_card.v1", - data=data, - ) - - -def build_action_legacy( - label: str, - action_type: str = "primary", - *, - target: str | None = None, - action: str | None = None, -) -> dict[str, Any]: - """Legacy wrapper for action.""" - result: dict[str, Any] = { - "type": action_type, - "label": label, - } - if target is not None: - result["target"] = target - if action is not None: - result["action"] = action - return result + return build_card(children, node_id=node_id, status=status) diff --git a/backend/src/schemas/messages/chat_message.py b/backend/src/schemas/messages/chat_message.py index 736b0a9..11f2d93 100644 --- a/backend/src/schemas/messages/chat_message.py +++ b/backend/src/schemas/messages/chat_message.py @@ -1,12 +1,14 @@ from __future__ import annotations from datetime import datetime +from decimal import Decimal from typing import ClassVar from uuid import UUID -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field +from schemas.agent.runtime_models import RouterAgentOutput, WorkerAgentOutputRich -from ..agent import AgentType, ToolAgentOutput, WorkerAgentOutput +from ..agent import AgentType, ToolAgentOutput class UserMessageAttachments(BaseModel): @@ -22,8 +24,9 @@ class AgentChatMessageMetadata(BaseModel): run_id: str agent_type: AgentType | None = None user_message_attachments: UserMessageAttachments | None = None + router_agent_output: RouterAgentOutput | None = None tool_agent_output: ToolAgentOutput | None = None - worker_agent_output: WorkerAgentOutput | None = None + worker_agent_output: WorkerAgentOutputRich | None = None class AgentChatMessage(BaseModel): @@ -35,5 +38,11 @@ class AgentChatMessage(BaseModel): seq: int role: str content: str + model_code: str | None = None + tool_name: str | None = None + input_tokens: int = Field(default=0, ge=0) + output_tokens: int = Field(default=0, ge=0) + cost: Decimal = Field(default=Decimal("0")) + latency_ms: int | None = Field(default=None, ge=0) metadata: AgentChatMessageMetadata | dict[str, object] | None = None timestamp: datetime diff --git a/backend/src/services/litellm/service.py b/backend/src/services/litellm/service.py index d900453..5ff86e7 100644 --- a/backend/src/services/litellm/service.py +++ b/backend/src/services/litellm/service.py @@ -118,6 +118,31 @@ class LiteLLMService: + normalized_completion_tokens * selected_tier.output_cost_per_token ) + def build_usage_metadata( + self, + *, + model: str, + usage_summary: dict[str, int] | None, + ) -> dict[str, Any]: + summary = usage_summary or {} + input_tokens = max(int(summary.get("input_tokens", 0) or 0), 0) + output_tokens = max(int(summary.get("output_tokens", 0) or 0), 0) + latency_ms = max(int(summary.get("latency_ms", 0) or 0), 0) + cached_prompt_tokens = max(int(summary.get("cached_prompt_tokens", 0) or 0), 0) + cost = self.calculate_cost( + model=model, + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + cached_prompt_tokens=cached_prompt_tokens, + ) + return { + "model": model, + "inputTokens": input_tokens, + "outputTokens": output_tokens, + "cost": cost, + "latencyMs": latency_ms, + } + def run_completion_with_cost( self, *, diff --git a/backend/src/v1/agent/dependencies.py b/backend/src/v1/agent/dependencies.py index 17eb159..ba79151 100644 --- a/backend/src/v1/agent/dependencies.py +++ b/backend/src/v1/agent/dependencies.py @@ -8,11 +8,6 @@ from redis.asyncio import Redis from sqlalchemy.ext.asyncio import AsyncSession from core.agentscope.events import RedisStreamBus -from core.agentscope.runtime.tasks import ( - run_command_task, - run_command_task_bulk, - run_command_task_critical, -) from core.agentscope.tools.tool_result_storage import ( create_tool_result_storage, ) @@ -48,6 +43,12 @@ class TaskiqQueueClient: @staticmethod def _select_queue_task(command: dict[str, object]) -> Any: + from core.agentscope.runtime.tasks import ( + run_command_task, + run_command_task_bulk, + run_command_task_critical, + ) + queue = str(command.get("queue", "default")).strip().lower() if queue == "critical": return run_command_task_critical diff --git a/backend/src/v1/agent/repository.py b/backend/src/v1/agent/repository.py index a0ab66b..8dc204f 100644 --- a/backend/src/v1/agent/repository.py +++ b/backend/src/v1/agent/repository.py @@ -1,6 +1,7 @@ from __future__ import annotations from datetime import date, datetime, time, timedelta, timezone +from decimal import Decimal from typing import Protocol from uuid import UUID, uuid4 @@ -218,6 +219,12 @@ class AgentRepository: "seq": int(message.seq), "role": role, "content": message.content, + "model_code": message.model_code, + "tool_name": message.tool_name, + "input_tokens": int(message.input_tokens or 0), + "output_tokens": int(message.output_tokens or 0), + "cost": str(message.cost if message.cost is not None else Decimal("0")), + "latency_ms": message.latency_ms, "metadata": message.metadata_json, "timestamp": message.created_at.astimezone(timezone.utc).isoformat(), } diff --git a/backend/src/v1/agent/router.py b/backend/src/v1/agent/router.py index c29362a..12d73bb 100644 --- a/backend/src/v1/agent/router.py +++ b/backend/src/v1/agent/router.py @@ -37,6 +37,7 @@ from v1.agent.schemas import ( AttachmentReference, AttachmentSignedUrlResponse, AttachmentUploadResponse, + HistorySnapshotResponse, TaskAcceptedResponse, ) from v1.agent.service import AgentService, asr_service @@ -219,13 +220,13 @@ async def stream_events( ) -@router.get("/history") +@router.get("/history", response_model=HistorySnapshotResponse) async def get_user_history_snapshot( service: Annotated[AgentService, Depends(get_agent_service)], current_user: Annotated[CurrentUser, Depends(get_current_user)], thread_id: str | None = Query(default=None, alias="threadId"), before: date | None = Query(default=None), -) -> dict[str, object]: +) -> HistorySnapshotResponse: return await service.get_user_history_snapshot( current_user=current_user, thread_id=thread_id, diff --git a/backend/src/v1/agent/schemas.py b/backend/src/v1/agent/schemas.py index e634364..349a04f 100644 --- a/backend/src/v1/agent/schemas.py +++ b/backend/src/v1/agent/schemas.py @@ -2,6 +2,8 @@ from __future__ import annotations from pydantic import BaseModel, ConfigDict, Field +from schemas.agent.ui_schema import UiSchemaRenderer + class TaskAcceptedResponse(BaseModel): model_config = ConfigDict(populate_by_name=True, serialize_by_alias=True) @@ -33,3 +35,35 @@ class AttachmentSignedUrlResponse(BaseModel): bucket: str path: str url: str + + +class HistoryMessage(BaseModel): + """History message schema for /history endpoint response.""" + + model_config = ConfigDict(populate_by_name=True, serialize_by_alias=True) + + id: str = Field(description="Message UUID") + seq: int = Field(description="Message sequence number") + role: str = Field(description="Message role: user | assistant | tool") + content: str = Field(description="Message text content") + url: str | None = Field( + default=None, + description="Temporary signed URL for user-attached images", + ) + ui_schema: UiSchemaRenderer | None = Field( + default=None, + description="Compiled UI schema from worker/tool ui_hints for frontend rendering", + ) + timestamp: str = Field(description="Message creation timestamp in ISO-8601 format") + + +class HistorySnapshotResponse(BaseModel): + """Response schema for GET /api/v1/agent/history""" + + model_config = ConfigDict(populate_by_name=True, serialize_by_alias=True) + + scope: str = Field(default="history_day") + thread_id: str | None = Field(default=None, alias="threadId") + day: str | None = None + has_more: bool = Field(default=False, alias="hasMore") + messages: list[HistoryMessage] = Field(default_factory=list) diff --git a/backend/src/v1/agent/service.py b/backend/src/v1/agent/service.py index 3d1f01e..ccf4c14 100644 --- a/backend/src/v1/agent/service.py +++ b/backend/src/v1/agent/service.py @@ -8,7 +8,7 @@ from typing import Any, Protocol from urllib.parse import urlparse import dashscope -from ag_ui.core import RunAgentInput, StateSnapshotEvent +from ag_ui.core import RunAgentInput from dashscope.audio.asr import Recognition, RecognitionCallback from fastapi import HTTPException from sqlalchemy.exc import IntegrityError @@ -21,6 +21,7 @@ from schemas.messages.chat_message import ( AgentChatMessageMetadata, UserMessageAttachments, ) +from v1.agent.schemas import HistorySnapshotResponse logger = get_logger(__name__) _ALLOWED_ATTACHMENT_MIME_TYPES = {"image/png", "image/jpeg", "image/webp"} @@ -416,27 +417,48 @@ class AgentService: thread_id: str, before: date | None, current_user: CurrentUser, - ) -> dict[str, object]: + ) -> HistorySnapshotResponse: + from schemas.messages.chat_message import AgentChatMessage + from v1.agent.utils import convert_message_to_history + from v1.agent.schemas import HistoryMessage + owner = await self._repository.get_session_owner(session_id=thread_id) ensure_session_owner(owner_id=owner, current_user=current_user) day_payload = await self._repository.get_history_day( session_id=thread_id, before=before, ) - snapshot = { - "scope": "history_day", - "threadId": thread_id, - "day": day_payload["day"] if day_payload else None, - "hasMore": day_payload["hasMore"] if day_payload else False, - "messages": day_payload["messages"] if day_payload else [], - } - event = StateSnapshotEvent(snapshot=snapshot).model_dump( - mode="json", - by_alias=True, - exclude_none=True, + + messages: list[HistoryMessage] = [] + if day_payload: + raw_messages = day_payload.get("messages") or [] + for msg_dict in raw_messages: + msg = AgentChatMessage.model_validate(msg_dict) + + signed_url: str | None = None + if self._attachment_storage and msg.metadata: + att = msg.metadata.user_message_attachments + if att: + signed_url = await self._attachment_storage.create_signed_url( + bucket=att.bucket, + path=att.path, + expires_in_seconds=self._SIGNED_URL_EXPIRES_IN_SECONDS, + ) + + converted = convert_message_to_history(msg, None) + if signed_url: + converted["url"] = signed_url + messages.append(HistoryMessage.model_validate(converted)) + + return HistorySnapshotResponse( + scope="history_day", + threadId=thread_id, + day=str(day_payload.get("day")) + if day_payload and day_payload.get("day") + else None, + hasMore=bool(day_payload.get("hasMore")) if day_payload else False, + messages=messages, ) - event["threadId"] = thread_id - return event async def get_user_history_snapshot( self, @@ -444,22 +466,20 @@ class AgentService: current_user: CurrentUser, thread_id: str | None, before: date | None, - ) -> dict[str, object]: + ) -> HistorySnapshotResponse: target_thread_id = thread_id if target_thread_id is None: target_thread_id = await self._repository.get_latest_session_id_for_user( user_id=str(current_user.id) ) if target_thread_id is None: - return StateSnapshotEvent( - snapshot={ - "scope": "history_day", - "threadId": None, - "day": None, - "hasMore": False, - "messages": [], - } - ).model_dump(mode="json", by_alias=True, exclude_none=True) + return HistorySnapshotResponse( + scope="history_day", + threadId=None, + day=None, + hasMore=False, + messages=[], + ) return await self.get_history_snapshot( thread_id=target_thread_id, before=before, diff --git a/backend/src/v1/agent/utils.py b/backend/src/v1/agent/utils.py new file mode 100644 index 0000000..2aa7103 --- /dev/null +++ b/backend/src/v1/agent/utils.py @@ -0,0 +1,149 @@ +""" +历史消息转换工具函数 + +将数据库中的原始消息转换为 API 响应的数据结构 +""" + +from collections.abc import Callable +from typing import Any + +from core.agentscope.runtime.ui_compiler import compile as compile_ui_hints +from schemas.messages.chat_message import ( + AgentChatMessage, + AgentChatMessageMetadata, + UserMessageAttachments, +) + + +def convert_message_to_history( + message: AgentChatMessage, + get_signed_url_fn: Callable[[str, str], str] | None = None, +) -> dict[str, Any]: + """ + 将 AgentChatMessage 转换为 HistoryMessage 格式 + + 转换规则： + - role=user: 读取 metadata.user_message_attachments，将 bucket 转临时访问 url + - role=tool: 读取 content 和 metadata.tool_agent_output.ui_hints，编译成 ui_schema + - role=assistant: 读取 metadata.worker_agent_output.ui_hints，编译成 ui_schema + """ + role = message.role + content = message.content + metadata = message.metadata + + url: str | None = None + ui_schema: dict[str, Any] | None = None + + if role == "user": + url = _convert_user_attachments(metadata, get_signed_url_fn) + + elif role == "tool": + ui_schema = _compile_tool_ui_hints(metadata) + + elif role == "assistant": + ui_schema = _compile_worker_ui_hints(metadata) + + result: dict[str, Any] = { + "id": str(message.id), + "seq": message.seq, + "role": role, + "content": content, + "timestamp": message.timestamp.isoformat(), + } + + if url: + result["url"] = url + + if ui_schema: + result["uiSchema"] = ui_schema + + return result + + +def _convert_user_attachments( + metadata: AgentChatMessageMetadata | dict[str, Any] | None, + get_signed_url_fn: Callable[[str, str], str] | None, +) -> str | None: + """转换用户附件为临时访问 URL""" + if not metadata: + return None + + if isinstance(metadata, AgentChatMessageMetadata): + attachments = metadata.user_message_attachments + else: + attachments_data = metadata.get("user_message_attachments") + if not attachments_data: + return None + attachments = UserMessageAttachments.model_validate(attachments_data) + + if not attachments or not get_signed_url_fn: + return None + + try: + return get_signed_url_fn( + {"bucket": attachments.bucket, "path": attachments.path} + ) + except Exception: + return None + + +def _compile_tool_ui_hints( + metadata: AgentChatMessageMetadata | dict[str, Any] | None, +) -> dict[str, Any] | None: + """编译 tool 消息的 ui_hints""" + if not metadata: + return None + + if isinstance(metadata, AgentChatMessageMetadata): + tool_output = metadata.tool_agent_output + else: + tool_output_data = metadata.get("tool_agent_output") + if not tool_output_data: + return None + from schemas.agent.runtime_models import ToolAgentOutput + + tool_output = ToolAgentOutput.model_validate(tool_output_data) + + if not tool_output: + return None + + ui_hints = tool_output.ui_hints + if not ui_hints: + return None + + try: + compiled = compile_ui_hints(ui_hints) + return compiled + except Exception: + return None + + +def _compile_worker_ui_hints( + metadata: AgentChatMessageMetadata | dict[str, Any] | None, +) -> dict[str, Any] | None: + """编译 assistant 消息的 worker ui_hints""" + if not metadata: + return None + + if isinstance(metadata, AgentChatMessageMetadata): + worker_output = metadata.worker_agent_output + else: + worker_output_data = metadata.get("worker_agent_output") + if not worker_output_data: + return None + from schemas.agent.runtime_models import WorkerAgentOutputRich + + worker_output = WorkerAgentOutputRich.model_validate(worker_output_data) + + if not worker_output: + return None + + ui_hints = worker_output.ui_hints + if not ui_hints: + return None + + try: + compiled = compile_ui_hints(ui_hints) + return compiled + except Exception: + return None diff --git a/backend/tests/unit/core/agentscope/events/test_pipeline.py b/backend/tests/unit/core/agentscope/events/test_pipeline.py index 4b68fa2..2d5fa2e 100644 --- a/backend/tests/unit/core/agentscope/events/test_pipeline.py +++ b/backend/tests/unit/core/agentscope/events/test_pipeline.py @@ -17,12 +17,13 @@ async def test_pipeline_orders_codec_persist_publish() -> None: class _Store: async def persist(self, event: dict[str, object]) -> None: calls.append("persist") - assert event["type"] == "RUN_STARTED" + assert event["id"] == "evt-1" class _Bus: async def publish(self, *, session_id: str, event: dict[str, object]) -> str: calls.append("publish") assert session_id == "thread-1" + assert event["type"] == "RUN_STARTED" return "1-0" pipeline = AgentScopeEventPipeline(codec=_Codec(), store=_Store(), bus=_Bus()) diff --git a/backend/tests/unit/core/agentscope/runtime/test_orchestrator.py b/backend/tests/unit/core/agentscope/runtime/test_orchestrator.py index 8202aa2..7bd48db 100644 --- a/backend/tests/unit/core/agentscope/runtime/test_orchestrator.py +++ b/backend/tests/unit/core/agentscope/runtime/test_orchestrator.py @@ -1,9 +1,9 @@ from __future__ import annotations from typing import Any -from uuid import UUID import pytest +from ag_ui.core import RunAgentInput from core.agentscope.runtime.orchestrator import AgentScopeRuntimeOrchestrator from schemas.user import UserContext, parse_profile_settings @@ -19,36 +19,9 @@ class _FakePipeline: class _FakeRunner: - def __init__(self) -> None: - self.last_user_input: str | list[dict[str, Any]] | None = None - - async def run_router_then_worker( - self, - *, - session, - user_context, - user_input, - router_toolkit, - worker_toolkit, - extra_context=None, - ) -> dict[str, Any]: - del session, user_context, router_toolkit, worker_toolkit, extra_context - self.last_user_input = user_input - return { - "worker": { - "status": "success", - "answer": "done", - "key_points": [], - "result_type": "summary", - "suggested_actions": [], - "error": None, - "response_metadata": { - "model": "qwen3.5-flash", - "inputTokens": 10, - "outputTokens": 5, - }, - } - } + async def execute(self, **kwargs: object) -> dict[str, Any]: + del kwargs + return {"worker": {"answer": "done"}} def _user_context() -> UserContext: @@ -56,34 +29,17 @@ def _user_context() -> UserContext: id="00000000-0000-0000-0000-000000000001", username="alice", email="alice@example.com", - avatar_url=None, - bio=None, settings=parse_profile_settings(None), ) -def _run_command_with_binary() -> Any: - from ag_ui.core import RunAgentInput - +def _run_input() -> RunAgentInput: return RunAgentInput.model_validate( { "threadId": "00000000-0000-0000-0000-000000000010", "runId": "run-1", "state": {}, - "messages": [ - { - "id": "u1", - "role": "user", - "content": [ - {"type": "text", "text": "看这张图"}, - { - "type": "binary", - "mimeType": "image/png", - "url": "https://example.com/signed.png", - }, - ], - } - ], + "messages": [{"id": "u1", "role": "user", "content": "hello"}], "tools": [], "context": [], "forwardedProps": {}, @@ -92,71 +48,18 @@ def _run_command_with_binary() -> Any: @pytest.mark.asyncio -async def test_orchestrator_maps_binary_to_model_image_url( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_orchestrator_emits_run_lifecycle_events() -> None: pipeline = _FakePipeline() - runner = _FakeRunner() - monkeypatch.setattr( - "core.agentscope.runtime.orchestrator.build_stage_toolkit", - lambda **_: None, - ) - orchestrator = AgentScopeRuntimeOrchestrator(pipeline=pipeline, runner=runner) - - await orchestrator.run( - thread_id="00000000-0000-0000-0000-000000000010", - run_id="run-1", - context_messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": "看这张图"}, - { - "type": "image", - "source": { - "type": "url", - "url": "https://example.com/signed.png", - }, - }, - ], - } - ], - owner_id=UUID("00000000-0000-0000-0000-000000000001"), - user_context=_user_context(), - session=None, + orchestrator = AgentScopeRuntimeOrchestrator( + pipeline=pipeline, runner=_FakeRunner() ) - assert isinstance(runner.last_user_input, list) - assert runner.last_user_input[0]["type"] == "text" - assert runner.last_user_input[1]["type"] == "image_url" - assert ( - runner.last_user_input[1]["image_url"]["url"] - == "https://example.com/signed.png" - ) - - -@pytest.mark.asyncio -async def test_orchestrator_emits_worker_output_on_text_end( - monkeypatch: pytest.MonkeyPatch, -) -> None: - pipeline = _FakePipeline() - runner = _FakeRunner() - monkeypatch.setattr( - "core.agentscope.runtime.orchestrator.build_stage_toolkit", - lambda **_: None, - ) - orchestrator = AgentScopeRuntimeOrchestrator(pipeline=pipeline, runner=runner) - - await orchestrator.run( - thread_id="00000000-0000-0000-0000-000000000010", - run_id="run-1", + result = await orchestrator.run( + run_input=_run_input(), context_messages=[], - owner_id=UUID("00000000-0000-0000-0000-000000000001"), user_context=_user_context(), - session=None, ) - emitted = [item["event"] for item in pipeline.events] - text_end = next(item for item in emitted if item.get("type") == "text.end") - assert text_end["data"]["workerAgentOutput"]["answer"] == "done" - assert any(item.get("type") == "run.finished" for item in emitted) + assert result["worker"]["answer"] == "done" + event_types = [item["event"]["type"] for item in pipeline.events] + assert event_types == ["run.started", "run.finished"] diff --git a/backend/tests/unit/core/agentscope/runtime/test_react_runner.py b/backend/tests/unit/core/agentscope/runtime/test_react_runner.py index f4de678..f677ebe 100644 --- a/backend/tests/unit/core/agentscope/runtime/test_react_runner.py +++ b/backend/tests/unit/core/agentscope/runtime/test_react_runner.py @@ -1,223 +1,201 @@ from __future__ import annotations -import json -from types import SimpleNamespace - import pytest +from ag_ui.core import RunAgentInput +from agentscope.message import Msg -from core.agentscope.schemas.system_agent_config import SystemAgentLLMConfig -from core.agentscope.runtime.config_loader import RuntimeStageConfig from core.agentscope.runtime.react_runner import ( AgentScopeReActRunner, - _chat_response_text, - _merge_stage_response_metadata, - _parse_json_text, - _to_litellm_model, + StageExecutionResult, + SystemAgentRuntimeConfig, ) +from schemas.agent.runtime_models import ( + RouterAgentOutput, + UiMode, + WorkerAgentOutputRich, +) +from schemas.agent.system_agent import AgentType, SystemAgentLLMConfig +from schemas.user.context import UserContext, parse_profile_settings -def _stage_config() -> RuntimeStageConfig: - return RuntimeStageConfig( - stage="intent", - model_code="qwen3.5-flash", - provider_name="dashscope", - llm_config=SystemAgentLLMConfig( - temperature=0.1, max_tokens=128, timeout_seconds=30 - ), +class _FakePipeline: + def __init__(self) -> None: + self.events: list[dict[str, object]] = [] + + async def emit(self, *, session_id: str, event: dict[str, object]) -> str: + self.events.append({"session_id": session_id, "event": event}) + return "1-0" + + +class _FakeSessionCtx: + def __init__(self, session: object) -> None: + self._session = session + + async def __aenter__(self) -> object: + return self._session + + async def __aexit__(self, exc_type, exc, tb) -> None: + del exc_type, exc, tb + + +def _user_context() -> UserContext: + return UserContext( + id="00000000-0000-0000-0000-000000000001", + username="alice", + email="alice@example.com", + settings=parse_profile_settings(None), ) -def test_to_litellm_model_keeps_prefixed_model() -> None: - assert ( - _to_litellm_model(provider_name="dashscope", model_code="openai/gpt-4o") - == "openai/gpt-4o" - ) - - -def test_to_litellm_model_uses_plain_model_name_when_unprefixed() -> None: - assert ( - _to_litellm_model(provider_name="dashscope", model_code="qwen3.5-flash") - == "qwen3.5-flash" - ) - - -def test_parse_json_text_supports_fenced_json() -> None: - parsed = _parse_json_text('```json\n{"route":"DIRECT_RESPONSE"}\n```') - assert parsed["route"] == "DIRECT_RESPONSE" - - -def test_parse_json_text_rejects_non_json() -> None: - with pytest.raises(json.JSONDecodeError): - _parse_json_text("not-json") - - -def test_chat_response_text_falls_back_to_choice_message_content() -> None: - response = SimpleNamespace( - content=None, - choices=[ - { - "message": { - "content": '{"assistant_text":"fallback","response_metadata":{}}' - } - } - ], - ) - - assert ( - _chat_response_text(response) - == '{"assistant_text":"fallback","response_metadata":{}}' - ) - - -@pytest.mark.asyncio -async def test_run_json_stage_wraps_json_decode_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - pytest.importorskip("agentscope") - import agentscope.agent as agent_module - import agentscope.formatter as formatter_module - import agentscope.memory as memory_module - - class _FakeAgent: - def __init__(self, **kwargs: object) -> None: - del kwargs - - async def __call__(self, _msg: object) -> object: - return SimpleNamespace(get_text_content=lambda: "not-json") - - monkeypatch.setattr(agent_module, "ReActAgent", _FakeAgent) - monkeypatch.setattr(formatter_module, "OpenAIChatFormatter", lambda: object()) - monkeypatch.setattr(memory_module, "InMemoryMemory", lambda: object()) - - runner = AgentScopeReActRunner() - monkeypatch.setattr(runner, "_build_model", lambda **_: object()) - - with pytest.raises(RuntimeError, match="agent output format invalid"): - await runner.run_json_stage( - stage_config=_stage_config(), - agent_name="intent-agent", - system_prompt="sys", - user_prompt="user", - toolkit=None, - ) - - -@pytest.mark.asyncio -async def test_run_json_stage_wraps_runtime_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - pytest.importorskip("agentscope") - import agentscope.agent as agent_module - import agentscope.formatter as formatter_module - import agentscope.memory as memory_module - - class _FakeAgent: - def __init__(self, **kwargs: object) -> None: - del kwargs - - async def __call__(self, _msg: object) -> object: - raise ValueError("boom") - - monkeypatch.setattr(agent_module, "ReActAgent", _FakeAgent) - monkeypatch.setattr(formatter_module, "OpenAIChatFormatter", lambda: object()) - monkeypatch.setattr(memory_module, "InMemoryMemory", lambda: object()) - - runner = AgentScopeReActRunner() - monkeypatch.setattr(runner, "_build_model", lambda **_: object()) - - with pytest.raises(RuntimeError, match="agent execution failed"): - await runner.run_json_stage( - stage_config=_stage_config(), - agent_name="intent-agent", - system_prompt="sys", - user_prompt="user", - toolkit=None, - ) - - -@pytest.mark.asyncio -async def test_run_json_stage_report_merges_usage_metadata( - monkeypatch: pytest.MonkeyPatch, -) -> None: - class _FakeLiteLLMService: - def run_completion_with_cost(self, **kwargs: object) -> object: - del kwargs - return SimpleNamespace( - response={ - "model": "dashscope/qwen3.5-flash", - "choices": [ - { - "message": { - "content": '{"assistant_text":"ok","response_metadata":{}}' - } - } - ], +def _run_input() -> RunAgentInput: + return RunAgentInput.model_validate( + { + "threadId": "00000000-0000-0000-0000-000000000010", + "runId": "run-1", + "state": {}, + "messages": [{"id": "u1", "role": "user", "content": "hello"}], + "tools": [ + { + "name": "calendar.read", + "description": "read", + "parameters": {"type": "object"}, }, - usage=SimpleNamespace( - prompt_tokens=9, - completion_tokens=4, - cost=0.006, - ), - ) + { + "name": "calendar-write", + "description": "write", + "parameters": {"type": "object"}, + }, + ], + "context": [], + "forwardedProps": {}, + } + ) + +def _router_output(*, ui_mode: UiMode) -> RouterAgentOutput: + return RouterAgentOutput.model_validate( + { + "normalized_task_input": { + "user_text": "hello", + "multimodal_summary": [], + }, + "key_entities": [], + "constraints": [], + "task_typing": {"primary": "knowledge", "secondary": []}, + "execution_mode": "onestep", + "result_typing": {"primary": "direct_answer", "secondary": []}, + "ui": { + "ui_mode": ui_mode.value, + "ui_decision_reason": "need structure" + if ui_mode == UiMode.RICH + else "plain text", + }, + } + ) + + +@pytest.mark.asyncio +async def test_execute_uses_router_ui_mode_to_select_worker_output_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: runner = AgentScopeReActRunner() + pipeline = _FakePipeline() + worker_model_holder: dict[str, type[object]] = {} + + class _CommitSession: + async def commit(self) -> None: + return None + + monkeypatch.setattr( + "core.agentscope.runtime.react_runner.AsyncSessionLocal", + lambda: _FakeSessionCtx(_CommitSession()), + ) monkeypatch.setattr( runner, - "_build_litellm_service", - lambda: _FakeLiteLLMService(), + "_build_toolkits", + lambda **kwargs: ("router-toolkit", "worker-toolkit"), ) - report_stage = RuntimeStageConfig( - stage="report", - model_code="qwen3.5-flash", - provider_name="dashscope", - llm_config=SystemAgentLLMConfig( - temperature=0.1, - max_tokens=128, - timeout_seconds=30, - ), - ) - payload = await runner.run_json_stage( - stage_config=report_stage, - agent_name="report-agent", - system_prompt="sys", - user_prompt="user", - toolkit=None, - ) - - metadata = payload["response_metadata"] - assert metadata["model"] == "dashscope/qwen3.5-flash" - assert metadata["inputTokens"] == 9 - assert metadata["outputTokens"] == 4 - assert metadata["cost"] == 0.006 - assert isinstance(metadata["latencyMs"], int) - assert metadata["latencyMs"] >= 0 - - -def test_merge_stage_response_metadata_estimates_cost_from_pricing( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr( - "core.agentscope.runtime.react_runner._estimate_cost_by_pricing", - lambda **kwargs: 0.0025, - ) - payload = _merge_stage_response_metadata( - payload={"route": "DIRECT_RESPONSE", "response_metadata": {}}, - stage_config=_stage_config(), - response=SimpleNamespace( - usage=SimpleNamespace( - prompt_tokens=12, - completion_tokens=8, + async def _load_system_agent_config(**kwargs): + return SystemAgentRuntimeConfig( + agent_type=kwargs["agent_type"], + model_code="qwen3.5-flash" + if kwargs["agent_type"] == AgentType.ROUTER + else "deepseek-chat", + llm_config=SystemAgentLLMConfig( + temperature=0.1, max_tokens=256, timeout_seconds=30 ), - model="qwen3.5-flash", - ), - latency_ms=50, - system_prompt="system", - user_prompt="user", - assistant_text='{"route":"DIRECT_RESPONSE"}', + ) + + monkeypatch.setattr(runner, "_load_system_agent_config", _load_system_agent_config) + + async def _run_router_stage(**kwargs): + return StageExecutionResult( + message=Msg(name="router", content="", role="assistant"), + payload=_router_output(ui_mode=UiMode.RICH).model_dump(mode="json"), + response_metadata={ + "model": "qwen3.5-flash", + "inputTokens": 12, + "outputTokens": 6, + "cost": 0.001, + "latencyMs": 50, + }, + ) + + monkeypatch.setattr(runner, "_run_router_stage", _run_router_stage) + + async def _persist_router_message(**kwargs) -> None: + assert kwargs["model_code"] == "qwen3.5-flash" + + monkeypatch.setattr(runner, "_persist_router_message", _persist_router_message) + + async def _run_worker_stage(**kwargs): + worker_model_holder["model"] = kwargs["worker_output_model"] + return StageExecutionResult( + message=Msg(name="worker", content="done", role="assistant"), + payload=WorkerAgentOutputRich.model_validate( + { + "status": "success", + "answer": "done", + "key_points": [], + "result_type": "direct_answer", + "suggested_actions": [], + "error": None, + "ui_hints": None, + } + ).model_dump(mode="json", exclude_none=True), + response_metadata={ + "model": "deepseek-chat", + "inputTokens": 8, + "outputTokens": 4, + "cost": 0.002, + "latencyMs": 40, + }, + ) + + monkeypatch.setattr(runner, "_run_worker_stage", _run_worker_stage) + + result = await runner.execute( + user_context=_user_context(), + context_messages=[], + pipeline=pipeline, + run_input=_run_input(), ) - metadata = payload["response_metadata"] - assert metadata["inputTokens"] == 12 - assert metadata["outputTokens"] == 8 - assert metadata["cost"] == 0.0025 + assert worker_model_holder["model"].__name__ == "WorkerAgentOutputRich" + event_types = [] + for item in pipeline.events: + event = item.get("event") + if isinstance(event, dict): + event_types.append(event.get("type")) + assert event_types == ["step.start", "step.finish", "step.start", "step.finish"] + assert result["router"]["ui"]["ui_mode"] == "rich" + assert result["worker"]["answer"] == "done" + + +def test_extract_tool_names_normalizes_client_tool_names() -> None: + runner = AgentScopeReActRunner() + + names = runner._extract_tool_names(_run_input()) + + assert names == {"calendar_read", "calendar_write"} diff --git a/backend/tests/unit/core/agentscope/runtime/test_tasks.py b/backend/tests/unit/core/agentscope/runtime/test_tasks.py index 910f937..6ab06f0 100644 --- a/backend/tests/unit/core/agentscope/runtime/test_tasks.py +++ b/backend/tests/unit/core/agentscope/runtime/test_tasks.py @@ -6,15 +6,17 @@ from uuid import uuid4 import pytest import core.agentscope.runtime.tasks as tasks_module +from schemas.user import UserContext, parse_profile_settings def _run_input_payload() -> dict[str, Any]: return { "threadId": str(uuid4()), "runId": "run-1", + "state": {}, "messages": [], "tools": [], - "context": {}, + "context": [], "forwardedProps": {}, } @@ -27,6 +29,16 @@ class _FakeSessionCtx: del exc_type, exc, tb +async def _fake_user_context(**kwargs: object) -> UserContext: + del kwargs + return UserContext( + id=str(uuid4()), + username="alice", + email="alice@example.com", + settings=parse_profile_settings(None), + ) + + @pytest.mark.asyncio async def test_run_agentscope_task_calls_runtime_run( monkeypatch: pytest.MonkeyPatch, @@ -56,6 +68,7 @@ async def test_run_agentscope_task_calls_runtime_run( _fake_get_redis_client, ) monkeypatch.setattr(tasks_module, "AsyncSessionLocal", lambda: _FakeSessionCtx()) + monkeypatch.setattr(tasks_module, "_build_user_context", _fake_user_context) monkeypatch.setattr( tasks_module, "_build_recent_context_messages", @@ -85,9 +98,12 @@ async def test_run_agentscope_task_includes_recent_context_messages( del kwargs async def run(self, **kwargs: object) -> object: - command = kwargs.get("command") - if command is not None: - raw_messages = getattr(command, "messages", []) + raw_context_messages = kwargs.get("context_messages") + raw_run_input = kwargs.get("run_input") + if isinstance(raw_context_messages, list): + captured_messages.extend(raw_context_messages) + if raw_run_input is not None: + raw_messages = getattr(raw_run_input, "messages", []) if isinstance(raw_messages, list): captured_messages.extend(raw_messages) return object() @@ -110,6 +126,7 @@ async def test_run_agentscope_task_includes_recent_context_messages( _fake_get_redis_client, ) monkeypatch.setattr(tasks_module, "AsyncSessionLocal", lambda: _FakeSessionCtx()) + monkeypatch.setattr(tasks_module, "_build_user_context", _fake_user_context) monkeypatch.setattr( tasks_module, "_build_recent_context_messages", @@ -133,7 +150,7 @@ async def test_run_agentscope_task_includes_recent_context_messages( assert len(captured_messages) == 2 assert captured_messages[0]["id"] == "ctx-1" - assert captured_messages[1]["id"] == "u1" + assert getattr(captured_messages[1], "id", None) == "u1" @pytest.mark.asyncio diff --git a/backend/tests/unit/core/agentscope/test_toolkit.py b/backend/tests/unit/core/agentscope/test_toolkit.py new file mode 100644 index 0000000..20f1a7e --- /dev/null +++ b/backend/tests/unit/core/agentscope/test_toolkit.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Any, cast +from uuid import uuid4 + +from core.agentscope.tools.toolkit import build_stage_toolkit +from schemas.agent.system_agent import AgentType + + +def test_build_stage_toolkit_filters_requested_tools_by_agent_type(monkeypatch) -> None: + captured: dict[str, object] = {} + + def _fake_build_toolkit(**kwargs): + captured.update(kwargs) + return object() + + monkeypatch.setattr( + "core.agentscope.tools.toolkit.build_toolkit", _fake_build_toolkit + ) + + build_stage_toolkit( + agent_type=AgentType.ROUTER, + session=cast(Any, object()), + owner_id=uuid4(), + enabled_tool_names={"calendar_read", "calendar_write", "user_lookup"}, + ) + + assert captured["enabled_tool_names"] == {"calendar_read", "user_lookup"} diff --git a/backend/tests/unit/services/test_litellm_service.py b/backend/tests/unit/services/test_litellm_service.py index 7025930..d03b4cf 100644 --- a/backend/tests/unit/services/test_litellm_service.py +++ b/backend/tests/unit/services/test_litellm_service.py @@ -60,3 +60,25 @@ def test_run_completion_extracts_usage_and_cost() -> None: assert result.usage.total_tokens == 2100 assert result.usage.cost == pytest.approx(0.00051) assert captured["response_format"] == {"type": "json_object"} + + +def test_build_usage_metadata_calculates_cost_from_usage_summary() -> None: + service = LiteLLMService() + + metadata = service.build_usage_metadata( + model="dashscope/qwen3.5-flash", + usage_summary={ + "input_tokens": 2000, + "output_tokens": 100, + "latency_ms": 321, + "cached_prompt_tokens": 500, + }, + ) + + assert metadata == { + "model": "dashscope/qwen3.5-flash", + "inputTokens": 2000, + "outputTokens": 100, + "cost": pytest.approx(0.00051), + "latencyMs": 321, + } diff --git a/docs/bugs/agent-history-messages.md b/docs/bugs/agent-history-messages.md deleted file mode 100644 index bf741ed..0000000 --- a/docs/bugs/agent-history-messages.md +++ /dev/null @@ -1,45 +0,0 @@ -# Agent 历史消息获取重构 - -## Bug 描述 - -`_build_recent_context_messages` 函数需要重构。 - -## 问题 - -当前实现直接使用 SQL 查询 `AgentChatMessage` 模型，但存在以下问题： - -1. **数据格式复杂**：落库后的消息包含多种角色（user/assistant/tool），content 可能是 dict 或 str -2. **需要转换**：需要将数据库模型正确转换为 Agent 可用的消息格式（符合 AG-UI Message 规范） -3. **Service 缺失**：没有使用 Repository/Service 模式，直接操作数据库 - -## 当前代码（已清空） - -位置：`src/core/agentscope/runtime/tasks.py` - -```python -async def _build_recent_context_messages( - *, - session: Any, - thread_id: str, - current_run_id: str, - max_messages: int = 20, -) -> list[dict[str, Any]]: - # TODO: 重新设计 - # 问题：落库后的消息包含多种角色（user/assistant/tool），需要正确转换为 Agent 可用的消息格式 - # 方案：使用 AgentRepository 或新建专门的 Service 方法来处理 - return [] -``` - -## 预期方案 - -1. 在 `AgentRepository` 中添加 `get_recent_messages` 方法 -2. 或创建专门的 `AgentMessageService` -3. 需要处理： - - 消息角色转换（user/assistant/tool -> AG-UI 格式） - - content 可能是 dict（包含 attachments 等）或 str - - 排除当前 run 的消息 - - 限制返回数量 - -## 状态 - -- [ ] 待处理 diff --git a/docs/plans/2026-03-12-home-composer-redesign-design.md b/docs/plans/2026-03-12-home-composer-redesign-design.md deleted file mode 100644 index 90e0c05..0000000 --- a/docs/plans/2026-03-12-home-composer-redesign-design.md +++ /dev/null @@ -1,122 +0,0 @@ -# Home 输入组件重做设计（HomeComposer Redesign） - -## 1. 目标与范围 - -### 1.1 目标 -- 解决当前输入组件“质感弱、结构割裂、录音时布局漂移”的问题。 -- 统一 `+` 按钮、输入区、右侧动作图标到一个圆角矩形主容器内。 -- 保留并复用现有录音、转写、自动发送、停止生成、Toast 错误处理逻辑。 - -### 1.2 非目标 -- 不改动聊天流、消息发送后端协议、语音识别接口。 -- 不改动 `+` 按钮业务行为。 -- 不新增独立的页面级浮层录音面板。 - -## 2. 问题诊断（现状） - -- 当前输入区由多个分离容器拼接，视觉上像“纸片贴上去”。 -- 输入框本体和右侧图标视觉上未合为一个整体容器。 -- “按住说话”提示与录音动画在主布局外追加，录音时造成结构上下跳动。 - -## 3. 方案对比 - -### 方案 A：单容器双模式（推荐） -- 单一胶囊主容器承载三段：左操作、中间主内容、右操作。 -- 中间区域在文本模式与按住说话模式之间替换（`AnimatedSwitcher`）。 -- 录音动画仅在中间区域内部切换显示，不改变主容器高度。 - -优点：结构稳定、状态清晰、维护成本低。 -缺点：视觉表达自由度略低于 Overlay 方案。 - -### 方案 B：双容器交叉切换 -- 文本容器和语音容器完整分离，做交叉淡入。 - -优点：切换动效可做得更明显。 -缺点：状态同步复杂，容易再次出现错位与边界问题。 - -### 方案 C：Overlay 浮层 -- 保持输入容器不变，录音时叠加浮层。 - -优点：动画自由度高。 -缺点：与“整块替换”诉求不一致，事件命中与无障碍处理更复杂。 - -结论：采用方案 A。 - -## 4. 信息架构与组件边界 - -## 4.1 新组件 -- 新建 `HomeComposer`（从 `home_screen.dart` 抽离输入区渲染职责）。 -- `HomeScreen` 继续持有业务状态与行为方法，`HomeComposer` 负责展示与手势分发。 - -## 4.2 主容器结构 -- 一个圆角矩形主容器（轻拟物胶囊风格）。 -- 左侧：`+` 按钮（行为不变）。 -- 中间： - - 文本模式：无边框输入区（文字垂直居中）。 - - 语音模式：按住说话按钮区。 - - 录音中/识别中：在语音模式内部替换状态内容。 -- 右侧：动作图标（声波/键盘/发送/停止）。 - -## 5. 状态机设计 - -## 5.1 状态定义 -- 模式层：`text` / `holdToSpeak` -- 过程层：`idle` / `recording` / `transcribing` - -## 5.2 核心约束 -- 主容器高度固定，状态变化不得引发外层布局高度变化。 -- `recording` 时禁止模式切换，避免状态错位。 - -## 5.3 右侧图标决策 -- Agent 等待中：停止图标。 -- 非等待： - - 有文本：发送图标。 - - 无文本且 `text`：`LucideIcons.activity`。 - - 无文本且 `holdToSpeak`：`LucideIcons.keyboard`。 - -## 6. 交互与动画 - -## 6.1 长按语音流程 -1. `onLongPressStart`：触发 `HapticFeedback.lightImpact()`，开始录音。 -2. `onLongPressMoveUpdate`：上滑超过阈值，取消录音。 -3. `onLongPressEnd`：未取消则停止录音，转写并自动发送。 - -## 6.2 提示文案显示策略 -- “松开发送，上滑取消”仅在 `recording` 显示。 -- 空闲按住说话模式不显示该提示。 - -## 6.3 动画策略 -- 模式切换：短时 `AnimatedSwitcher`（淡入/轻位移）。 -- 录音波形：仅在 `recording` 驱动；停止后立刻回收。 -- 动画渲染在中间区域内部，不新增外部占位。 - -## 7. 视觉规范（Design Tokens） - -- 严格使用 `apps/lib/core/theme/design_tokens.dart` 中的 `AppColors`、`AppSpacing`、`AppRadius`。 -- 禁止硬编码颜色、间距、圆角、尺寸、阴影。 -- 主容器采用白色底 + 细边 + 柔和阴影，形成轻拟物层次。 -- 输入区内部无额外边框，确保文字垂直居中和图标视觉对齐。 - -## 8. 验收标准 - -- 图标与输入区在同一圆角矩形中，不再分离。 -- 录音全流程不出现输入组件上移、下坠或高度抖动。 -- 提示文案仅在实际录音中显示。 -- 文本/语音模式切换平滑；`+` 与发送逻辑行为保持一致。 - -## 9. 风险与缓解 - -- 风险：重构过程中影响现有发送/停止生成分支。 - - 缓解：优先复用原有行为方法，仅调整 UI 结构与状态映射。 -- 风险：手势与模式切换并发导致状态错乱。 - - 缓解：录音期间加切换锁，结束后释放。 - -## 10. 验证计划 - -- 手工验证： - - 文本发送、停止生成、`+` 弹层、语音长按录音、上滑取消、自动发送。 - - 文本模式与语音模式往返切换稳定性。 -- Widget 测试（建议新增）： - - 右侧图标状态映射测试。 - - 录音中提示文案显示条件测试。 - - 模式切换时主容器高度恒定测试。 diff --git a/docs/plans/2026-03-12-home-composer-redesign-implementation-plan.md b/docs/plans/2026-03-12-home-composer-redesign-implementation-plan.md deleted file mode 100644 index 0c16542..0000000 --- a/docs/plans/2026-03-12-home-composer-redesign-implementation-plan.md +++ /dev/null @@ -1,275 +0,0 @@ -# Home Composer Redesign Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** 重做 Home 输入组件，统一为单胶囊容器并稳定语音长按交互，消除布局漂移，同时保持 `+` 与发送等既有业务逻辑不变。 - -**Architecture:** 采用“单容器双模式”方案：`HomeScreen` 继续持有业务状态与动作，新增 `HomeComposer` 专注 UI 与手势分发；中间区域用受控状态在文本/按住说话/录音中/识别中之间切换，外层高度固定。录音提示与声波动画都内聚在主容器内部渲染，避免额外布局占位。 - -**Tech Stack:** Flutter, flutter_bloc, lucide_icons, design tokens (`AppColors`/`AppSpacing`/`AppRadius`), widget test - ---- - -### Task 1: 建立 HomeComposer 组件骨架与参数契约 - -**Files:** -- Create: `apps/lib/features/home/ui/widgets/home_composer.dart` -- Modify: `apps/lib/features/home/ui/screens/home_screen.dart` -- Test: `apps/test/features/home/ui/widgets/home_composer_test.dart` - -**Step 1: 写失败测试（渲染结构）** - -```dart -testWidgets('renders one unified rounded composer container', (tester) async { - // pump HomeComposer with minimum required callbacks/states - // expect: one root container, plus button, center content slot, right action slot -}); -``` - -**Step 2: 运行测试确认失败** - -Run: `flutter test test/features/home/ui/widgets/home_composer_test.dart --plain-name "renders one unified rounded composer container"` -Expected: FAIL（`HomeComposer` 未定义或结构不匹配） - -**Step 3: 写最小实现** - -```dart -class HomeComposer extends StatelessWidget { - const HomeComposer({ - super.key, - required this.isHoldToSpeakMode, - required this.isRecording, - required this.isTranscribing, - required this.hasMessage, - required this.isWaitingAgent, - required this.onTapPlus, - required this.onTapRightAction, - required this.centerChild, - }); - // unified capsule container with left/center/right slots -} -``` - -**Step 4: 再跑测试确认通过** - -Run: `flutter test test/features/home/ui/widgets/home_composer_test.dart --plain-name "renders one unified rounded composer container"` -Expected: PASS - -**Step 5: 小步提交（仅用户明确要求时）** - -```bash -git add apps/lib/features/home/ui/widgets/home_composer.dart apps/lib/features/home/ui/screens/home_screen.dart apps/test/features/home/ui/widgets/home_composer_test.dart -git commit -m "refactor: extract unified home composer container" -``` - -### Task 2: 完成右侧图标状态映射（activity/keyboard/send/stop） - -**Files:** -- Modify: `apps/lib/features/home/ui/widgets/home_composer.dart` -- Modify: `apps/lib/features/home/ui/screens/home_screen.dart` -- Test: `apps/test/features/home/ui/widgets/home_composer_test.dart` - -**Step 1: 写失败测试（图标状态机）** - -```dart -testWidgets('right action icon follows state priority', (tester) async { - // waiting > hasMessage > holdToSpeakMode > textMode - // expect LucideIcons.square/send/keyboard/activity respectively -}); -``` - -**Step 2: 运行测试确认失败** - -Run: `flutter test test/features/home/ui/widgets/home_composer_test.dart --plain-name "right action icon follows state priority"` -Expected: FAIL（图标选择逻辑尚未完整实现） - -**Step 3: 最小实现图标决策** - -```dart -IconData resolveRightIcon(...) { - if (isWaitingAgent) return LucideIcons.square; - if (hasMessage) return LucideIcons.send; - return isHoldToSpeakMode ? LucideIcons.keyboard : LucideIcons.activity; -} -``` - -**Step 4: 再跑测试确认通过** - -Run: `flutter test test/features/home/ui/widgets/home_composer_test.dart --plain-name "right action icon follows state priority"` -Expected: PASS - -**Step 5: 小步提交（仅用户明确要求时）** - -```bash -git add apps/lib/features/home/ui/widgets/home_composer.dart apps/lib/features/home/ui/screens/home_screen.dart apps/test/features/home/ui/widgets/home_composer_test.dart -git commit -m "refactor: stabilize composer right action icon mapping" -``` - -### Task 3: 实现中间区域双模式替换并固定高度 - -**Files:** -- Modify: `apps/lib/features/home/ui/widgets/home_composer.dart` -- Modify: `apps/lib/features/home/ui/screens/home_screen.dart` -- Test: `apps/test/features/home/ui/widgets/home_composer_test.dart` - -**Step 1: 写失败测试（模式切换不改变高度）** - -```dart -testWidgets('composer height remains stable across mode switches', (tester) async { - // measure size in text mode and hold-to-speak mode - // expect equal heights -}); -``` - -**Step 2: 运行测试确认失败** - -Run: `flutter test test/features/home/ui/widgets/home_composer_test.dart --plain-name "composer height remains stable across mode switches"` -Expected: FAIL（当前结构切换时高度波动） - -**Step 3: 最小实现（AnimatedSwitcher + fixed constraints）** - -```dart -SizedBox( - height: composerHeight, - child: AnimatedSwitcher( - duration: switchDuration, - child: isHoldToSpeakMode ? holdToSpeakChild : textInputChild, - ), -) -``` - -**Step 4: 再跑测试确认通过** - -Run: `flutter test test/features/home/ui/widgets/home_composer_test.dart --plain-name "composer height remains stable across mode switches"` -Expected: PASS - -**Step 5: 小步提交（仅用户明确要求时）** - -```bash -git add apps/lib/features/home/ui/widgets/home_composer.dart apps/lib/features/home/ui/screens/home_screen.dart apps/test/features/home/ui/widgets/home_composer_test.dart -git commit -m "refactor: keep composer layout stable during mode switch" -``` - -### Task 4: 实现长按录音交互（开始/上滑取消/松开发送） - -**Files:** -- Modify: `apps/lib/features/home/ui/screens/home_screen.dart` -- Modify: `apps/lib/features/home/ui/widgets/home_composer.dart` -- Test: `apps/test/features/home/ui/widgets/home_composer_test.dart` - -**Step 1: 写失败测试（录音提示只在 recording）** - -```dart -testWidgets('recording hint appears only while recording', (tester) async { - // idle hold-to-speak: no hint - // recording: show "松开发送，上滑取消" -}); -``` - -**Step 2: 运行测试确认失败** - -Run: `flutter test test/features/home/ui/widgets/home_composer_test.dart --plain-name "recording hint appears only while recording"` -Expected: FAIL - -**Step 3: 最小实现录音流程映射** - -```dart -onLongPressStart => HapticFeedback.lightImpact() + onHoldStart(); -onLongPressMoveUpdate => if (dy < threshold) onHoldCancel(); -onLongPressEnd => onHoldEnd(autoSend: true); -``` - -**Step 4: 再跑测试确认通过** - -Run: `flutter test test/features/home/ui/widgets/home_composer_test.dart --plain-name "recording hint appears only while recording"` -Expected: PASS - -**Step 5: 小步提交（仅用户明确要求时）** - -```bash -git add apps/lib/features/home/ui/widgets/home_composer.dart apps/lib/features/home/ui/screens/home_screen.dart apps/test/features/home/ui/widgets/home_composer_test.dart -git commit -m "feat: rework hold-to-speak interaction with stable recording state" -``` - -### Task 5: 视觉重构为轻拟物胶囊（仅 tokens） - -**Files:** -- Modify: `apps/lib/features/home/ui/widgets/home_composer.dart` -- Modify: `apps/lib/core/theme/design_tokens.dart`（仅当现有 token 不足时新增） -- Test: `apps/test/features/home/ui/widgets/home_composer_test.dart` - -**Step 1: 写失败测试（主容器统一性）** - -```dart -testWidgets('plus, center and right action are inside same capsule', (tester) async { - // find one capsule host and verify children are descendants -}); -``` - -**Step 2: 运行测试确认失败** - -Run: `flutter test test/features/home/ui/widgets/home_composer_test.dart --plain-name "plus, center and right action are inside same capsule"` -Expected: FAIL - -**Step 3: 最小视觉实现（不硬编码）** - -```dart -// use AppColors/AppSpacing/AppRadius and existing shadow tokens -// no hardcoded color/spacing/radius/size -``` - -**Step 4: 再跑测试确认通过** - -Run: `flutter test test/features/home/ui/widgets/home_composer_test.dart --plain-name "plus, center and right action are inside same capsule"` -Expected: PASS - -**Step 5: 小步提交（仅用户明确要求时）** - -```bash -git add apps/lib/features/home/ui/widgets/home_composer.dart apps/lib/core/theme/design_tokens.dart apps/test/features/home/ui/widgets/home_composer_test.dart -git commit -m "refactor: redesign home composer with neumorphic capsule style" -``` - -### Task 6: 集成回归与文档同步 - -**Files:** -- Modify: `apps/lib/features/home/ui/screens/home_screen.dart` -- Modify: `docs/runtime/runtime-route.md`（若交互说明有变化） - -**Step 1: 运行目标测试文件** - -Run: `flutter test test/features/home/ui/widgets/home_composer_test.dart` -Expected: PASS - -**Step 2: 运行 Home 相关回归测试（若新增）** - -Run: `flutter test test/features/home` -Expected: PASS（若目录存在） - -**Step 3: 运行应用侧基础回归** - -Run: `flutter test` -Expected: PASS 或仅存在与本改动无关的已知失败 - -**Step 4: 记录验证结论** - -```text -- 输入组件统一容器：通过 -- 模式切换稳定：通过 -- 录音提示条件：通过 -- + 按钮/发送逻辑：通过 -``` - -**Step 5: 小步提交（仅用户明确要求时）** - -```bash -git add apps/lib/features/home/ui/screens/home_screen.dart apps/lib/features/home/ui/widgets/home_composer.dart apps/test/features/home/ui/widgets/home_composer_test.dart docs/runtime/runtime-route.md -git commit -m "test: add regression coverage for home composer redesign" -``` - -## 实施注意事项 - -- 保持 `HomeScreen` 作为业务状态单一来源，避免在 `HomeComposer` 内部复制业务状态。 -- 录音中 (`recording`) 禁止触发模式切换，防止并发手势引发错位。 -- 严格遵守 `apps/AGENTS.md`：不硬编码视觉值，必须使用 design tokens。 -- 用户反馈统一使用 `Toast.show(...)`，不得引入 `SnackBar`。 diff --git a/docs/plans/2026-03-13-agent-runs-multimodal-implementation.md b/docs/plans/2026-03-13-agent-runs-multimodal-implementation.md deleted file mode 100644 index 82965dd..0000000 --- a/docs/plans/2026-03-13-agent-runs-multimodal-implementation.md +++ /dev/null @@ -1,261 +0,0 @@ -# Agent Runs Multimodal Refactor Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** 让 runs/resume 使用真实多模态图片输入，并将 worker/tool 按新结构化 metadata 规范落库。 - -**Architecture:** 保持现有 event pipeline，不引入旁路写库。请求入口完成 URL 安全边界校验；runtime 将 `binary` 转模型可识别 `image_url` block；event store 统一校验 `WorkerAgentOutput` / `ToolAgentOutput` 并完成 `content` 映射。 - -**Tech Stack:** FastAPI, Pydantic v2, SQLAlchemy AsyncSession, AgentScope, LiteLLM, Redis Stream - ---- - -### Task 1: Runs 输入安全边界 - -**Files:** -- Modify: `backend/src/core/agentscope/schemas/agui_input.py` -- Modify: `backend/src/v1/agent/router.py` -- Modify: `backend/src/v1/agent/service.py` -- Test: `backend/tests/unit/v1/agent/test_agent_router.py` - -**Step 1: Write the failing test** - -```python -def test_runs_rejects_non_project_signed_url(...) -> None: - payload = build_run_payload_with_binary_url("https://evil.example.com/storage/v1/object/sign/..." ) - resp = client.post("/api/v1/agent/runs", json=payload, headers=auth_headers) - assert resp.status_code == 422 -``` - -**Step 2: Run test to verify it fails** - -Run: `pytest backend/tests/unit/v1/agent/test_agent_router.py::test_runs_rejects_non_project_signed_url -v` -Expected: FAIL（当前不会拦截该 URL） - -**Step 3: Write minimal implementation** - -```python -def validate_binary_signed_url_scope(*, url: str, user_id: UUID, thread_id: UUID) -> tuple[str, str]: - bucket, path = supabase_service.parse_signed_url(url) - # check host, bucket, path prefix agent-inputs/{user_id}/{thread_id}/uploads/ - return bucket, path -``` - -在 `runs/resume` 请求入口调用校验；若请求含 binary 且当前模型不支持视觉，抛 `HTTPException(status_code=422, ...)`。 - -**Step 4: Run test to verify it passes** - -Run: `pytest backend/tests/unit/v1/agent/test_agent_router.py::test_runs_rejects_non_project_signed_url -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add backend/src/core/agentscope/schemas/agui_input.py backend/src/v1/agent/router.py backend/src/v1/agent/service.py backend/tests/unit/v1/agent/test_agent_router.py -git commit -m "fix: enforce signed image url scope on runs" -``` - -### Task 2: Runtime 多模态直传（移除文本化图片） - -**Files:** -- Modify: `backend/src/core/agentscope/runtime/orchestrator.py` -- Modify: `backend/src/core/agentscope/prompts/agent_prompt.py` -- Test: `backend/tests/unit/core/agentscope/runtime/test_orchestrator.py` - -**Step 1: Write the failing test** - -```python -async def test_orchestrator_passes_image_url_block_to_runner() -> None: - command = build_run_input_with_binary("https://project.supabase.co/storage/v1/object/sign/...") - await orchestrator.run(..., command=command, ...) - assert fake_runner.user_input[1]["type"] == "image_url" -``` - -**Step 2: Run test to verify it fails** - -Run: `pytest backend/tests/unit/core/agentscope/runtime/test_orchestrator.py::test_orchestrator_passes_image_url_block_to_runner -v` -Expected: FAIL（当前路径仍可能文本化） - -**Step 3: Write minimal implementation** - -```python -def _to_model_multimodal_blocks(content_blocks: list[dict[str, Any]]) -> list[dict[str, Any]]: - # text -> {type:"text", text:...} - # binary -> {type:"image_url", image_url:{url:...}} -``` - -将 runner 输入改为上述多模态块；禁止把图片块拼进普通字符串。 - -**Step 4: Run test to verify it passes** - -Run: `pytest backend/tests/unit/core/agentscope/runtime/test_orchestrator.py::test_orchestrator_passes_image_url_block_to_runner -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add backend/src/core/agentscope/runtime/orchestrator.py backend/src/core/agentscope/prompts/agent_prompt.py backend/tests/unit/core/agentscope/runtime/test_orchestrator.py -git commit -m "feat: pass image blocks as multimodal payload to model" -``` - -### Task 3: Worker 结构化落库（content=answer） - -**Files:** -- Modify: `backend/src/core/agentscope/events/store.py` -- Modify: `backend/src/core/agentscope/runtime/orchestrator.py` -- Test: `backend/tests/unit/core/agentscope/events/test_store.py` - -**Step 1: Write the failing test** - -```python -async def test_text_message_end_persists_worker_output_and_answer_content() -> None: - event = build_text_end_event(worker_agent_output={"answer": "ok", ...}) - await store.persist(event) - assert saved.content == "ok" - assert saved.metadata_json["worker_agent_output"]["answer"] == "ok" -``` - -**Step 2: Run test to verify it fails** - -Run: `pytest backend/tests/unit/core/agentscope/events/test_store.py::test_text_message_end_persists_worker_output_and_answer_content -v` -Expected: FAIL - -**Step 3: Write minimal implementation** - -```python -worker = WorkerAgentOutput.model_validate(event.get("workerAgentOutput") or {}) -content = worker.answer -metadata["worker_agent_output"] = worker.model_dump(mode="json") -``` - -orchestrator 在 `text.end` 事件 data 写入 `workerAgentOutput`。 - -**Step 4: Run test to verify it passes** - -Run: `pytest backend/tests/unit/core/agentscope/events/test_store.py::test_text_message_end_persists_worker_output_and_answer_content -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add backend/src/core/agentscope/events/store.py backend/src/core/agentscope/runtime/orchestrator.py backend/tests/unit/core/agentscope/events/test_store.py -git commit -m "refactor: persist worker output schema with answer as message content" -``` - -### Task 4: Tool 结构化落库（content=result_summary）并删除旧摘要逻辑 - -**Files:** -- Modify: `backend/src/core/agentscope/events/store.py` -- Modify: `backend/src/core/agentscope/runtime/orchestrator.py` -- Delete: `backend/src/core/agentscope/events/tool_result_summary.py` -- Test: `backend/tests/unit/core/agentscope/events/test_store.py` - -**Step 1: Write the failing test** - -```python -async def test_tool_result_persists_tool_output_and_summary_content() -> None: - event = build_tool_result_event(tool_agent_output={"result_summary": "done", ...}) - await store.persist(event) - assert saved.content == "done" - assert saved.metadata_json["tool_agent_output"]["result_summary"] == "done" -``` - -**Step 2: Run test to verify it fails** - -Run: `pytest backend/tests/unit/core/agentscope/events/test_store.py::test_tool_result_persists_tool_output_and_summary_content -v` -Expected: FAIL - -**Step 3: Write minimal implementation** - -```python -tool = ToolAgentOutput.model_validate(event.get("toolAgentOutput") or {}) -content = tool.result_summary -metadata["tool_agent_output"] = tool.model_dump(mode="json") -``` - -移除 `build_tool_content_summary` 相关 import/调用。 - -**Step 4: Run test to verify it passes** - -Run: `pytest backend/tests/unit/core/agentscope/events/test_store.py::test_tool_result_persists_tool_output_and_summary_content -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add backend/src/core/agentscope/events/store.py backend/src/core/agentscope/runtime/orchestrator.py backend/tests/unit/core/agentscope/events/test_store.py backend/src/core/agentscope/events/tool_result_summary.py -git commit -m "refactor: persist tool output schema and remove legacy summary builder" -``` - -### Task 5: Worker output 模型别名收敛（可选第二阶段） - -**Files:** -- Modify: `backend/src/schemas/agent/runtime_models.py` -- Modify: `backend/src/schemas/messages/chat_message.py` -- Test: `backend/tests/unit/schemas/agent/test_runtime_models.py` - -**Step 1: Write the failing test** - -```python -def test_worker_output_lite_disallows_ui_hints() -> None: - with pytest.raises(ValidationError): - WorkerAgentOutputLite.model_validate({... , "ui_hints": {...}}) -``` - -**Step 2: Run test to verify it fails** - -Run: `pytest backend/tests/unit/schemas/agent/test_runtime_models.py::test_worker_output_lite_disallows_ui_hints -v` -Expected: 根据现状决定（若已 fail 则作为守护测试） - -**Step 3: Write minimal implementation** - -```python -WorkerAgentOutput = WorkerAgentOutputLite | WorkerAgentOutputRich -``` - -如不想扩大变更，可保留现状并仅补充注释说明由 `resolve_worker_output_model` 决定运行时约束。 - -**Step 4: Run test to verify it passes** - -Run: `pytest backend/tests/unit/schemas/agent/test_runtime_models.py -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add backend/src/schemas/agent/runtime_models.py backend/src/schemas/messages/chat_message.py backend/tests/unit/schemas/agent/test_runtime_models.py -git commit -m "refactor: clarify worker output model contract for lite and rich modes" -``` - -### Task 6: 端到端回归与文档同步 - -**Files:** -- Modify: `docs/protocols/agent-chat-messages.md` -- Modify: `docs/runtime/runtime-route.md` - -**Step 1: Run targeted backend tests** - -Run: `pytest backend/tests/unit/v1/agent/test_agent_router.py backend/tests/unit/core/agentscope/runtime/test_orchestrator.py backend/tests/unit/core/agentscope/events/test_store.py -v` -Expected: PASS - -**Step 2: Run lint/type checks** - -Run: `cd backend && ruff check src tests && mypy src` -Expected: PASS - -**Step 3: Update docs for new contracts** - -- 明确 `runs` 的 URL 安全边界与 422 错误码。 -- 明确 `worker_agent_output`/`tool_agent_output` 的落库契约及 `content` 映射规则。 - -**Step 4: Final verification** - -Run: `pytest backend/tests -q` -Expected: PASS - -**Step 5: Commit** - -```bash -git add docs/protocols/agent-chat-messages.md docs/runtime/runtime-route.md -git commit -m "docs: align runs multimodal and structured persistence contracts" -``` diff --git a/docs/plans/2026-03-13-agent-runs-multimodal-storage-design.md b/docs/plans/2026-03-13-agent-runs-multimodal-storage-design.md deleted file mode 100644 index 4d875ad..0000000 --- a/docs/plans/2026-03-13-agent-runs-multimodal-storage-design.md +++ /dev/null @@ -1,87 +0,0 @@ -# Agent Runs Multimodal 与落库重构设计 - -**目标**：让 `POST /agent/runs` 支持真实多模态直传到模型（非文本化），并将 worker/tool 结果按新 metadata 协议结构化落库。 - -**范围**：后端 `runs/resume` 请求校验、runtime 输入转换、事件落库、history 回放一致性。 - ---- - -## 1. 背景与问题 - -- 当前 `binary` 内容在运行链路中被当作普通 JSON 文本拼接进入 prompt，模型拿不到原生图像输入。 -- tool 落库仍依赖旧摘要逻辑 `build_tool_content_summary`，与最新 `ToolAgentOutput` 元数据规范不一致。 -- worker 落库当前只落文本内容，未确保 `WorkerAgentOutput` 结构化对象与 `content=answer` 的一致关系。 - ---- - -## 2. 设计原则 - -- 协议单一信源：严格遵循 `docs/protocols/agent-chat-messages.md`，只接受 `binary` 形态，不兼容旧形态。 -- 最小安全边界：仅允许本项目 Supabase 私有桶签名 URL，拒绝任意外部 URL。 -- 事件驱动持久化：以 event store 作为唯一落库入口，避免双轨逻辑。 -- 数据可回放：history 始终可按 metadata 重新签名并回填 user 附件。 - ---- - -## 3. 目标数据流 - -1. `runs` 入参校验通过后，user message 入库（附件仅存 bucket/path/mime）。 -2. runtime 执行时，将 `binary` 转为模型多模态 `image_url` content block 直传。 -3. orchestrator 产出结构化事件： - - worker 主响应通过 `TEXT_MESSAGE_*` 事件发送，`TEXT_MESSAGE_END` 携带 `workerAgentOutput`。 - - tool 执行结果通过 `TOOL_CALL_RESULT` 事件发送，携带 `toolAgentOutput`。 -4. event store 统一校验并落库： - - worker：`content = answer`，metadata 写 `worker_agent_output`。 - - tool：`content = result_summary`，metadata 写 `tool_agent_output`。 -5. history 读取 user metadata 重新签名 URL，返回 `binary` block 给前端。 - ---- - -## 4. 安全与错误策略 - -### 4.1 URL 安全边界 - -- `binary.url` 必须满足： - - host 为当前 Supabase 项目域名。 - - path 为 `/storage/v1/object/sign/{bucket}/{path}`。 - - `{bucket}` 等于 `config.storage.bucket`。 - - `{path}` 前缀匹配 `agent-inputs/{user_id}/{thread_id}/uploads/`。 - -### 4.2 运行失败 - -- 保持 AG-UI 生命周期完整：`RUN_STARTED` 后只能 `RUN_FINISHED` 或 `RUN_ERROR` 结束。 -- 运行错误时不落半结构化消息，避免脏元数据。 - ---- - -## 5. 落库契约 - -### 5.1 Worker - -- 入库角色：`assistant` -- `messages.content = worker_agent_output.answer` -- `messages.metadata.worker_agent_output = WorkerAgentOutput`（完整、schema 校验后） - -### 5.2 Tool - -- 入库角色：`tool` -- `messages.content = tool_agent_output.result_summary` -- `messages.metadata.tool_agent_output = ToolAgentOutput`（完整、schema 校验后） -- 删除旧摘要逻辑：`build_tool_content_summary` - ---- - -## 6. 兼容性策略 - -- 不兼容旧输入块形态（如 `image_url` 作为 runs 输入）。 -- 历史接口输出协议保持不变，前端无需修改消费协议。 -- 原有 user 附件回放路径保留，只强化入站 URL 校验。 - ---- - -## 7. 验收标准 - -- runs 包含合法 `binary` 时，模型收到多模态消息（非文本化 JSON）。 -- 非本项目签名 URL 返回 `422`。 -- worker/tool 落库满足 `content` 与结构化 metadata 一一对应。 -- history 仍能正确回放 user 附件（临时签名 URL）。 diff --git a/docs/plans/2026-03-13-auth-pages-redesign-design.md b/docs/plans/2026-03-13-auth-pages-redesign-design.md deleted file mode 100644 index 7e688d1..0000000 --- a/docs/plans/2026-03-13-auth-pages-redesign-design.md +++ /dev/null @@ -1,318 +0,0 @@ -# Auth Pages And Feedback System Redesign - -## Goal - -Redesign the mobile authentication experience so the login, register, and reset-password flows feel like a polished assistant product instead of a flat form flow, while preserving existing business logic, routing, validation, and feedback behavior semantics. - -## Scope - -- Rebuild the UI for `login`, `register`, and `reset-password` -- Preserve existing auth logic, cubits, navigation, and API calls -- Redesign the fixed-length code input experience for verification code and invite code -- Redesign the feedback system into two coordinated layers: - - global floating toast messages - - in-component contextual messages -- Keep all work inside the existing Flutter design system and shared widget architecture - -## Constraints - -- Must follow `apps/AGENTS.md` -- Must follow `apps/rules/visual_design_language.md` -- Must use tokens from `apps/lib/core/theme/design_tokens.dart` -- Must not introduce a parallel feedback system outside the approved Toast and inline message architecture -- Must not change auth protocols, routing semantics, or submission logic - -## Current Problems - -### Visual Problems - -- The screens read as flat white pages with blue buttons, which matches a prohibited anti-pattern in the visual design language. -- The main CTA button uses color as its only emphasis mechanism, so it feels plastic and low-fidelity. -- The three auth screens share only superficial consistency, not a strong surface system. -- The reset-password screen presents all steps at once, causing poor rhythm and weak hierarchy. -- The verification code layout feels cramped and improvised because the code cells and send button compete on the same row. - -### Feedback Problems - -- Global toast messages are visually generic and lack product identity. -- Component-level messaging is under-specified and inconsistently expressed. -- Result messages and contextual validation messages are not clearly separated in responsibility. - -## Design Direction - -The approved direction is a floating-card auth system with a soft blue atmospheric background, a clear brand anchor, and a calm layered surface hierarchy. - -The target feeling is: - -- premium -- calm -- trustworthy -- assistant-oriented -- softly tactile -- mobile-native - -The redesign should feel like a cohesive product surface, not a stack of form containers. - -## Surface Model - -Each auth screen uses the same four-layer structure. - -### 1. Background Surface - -The screen background is not a plain blank fill. It should feel like a soft spatial field with subtle blue-gray atmosphere. This establishes the assistant mood before the user interacts with any form element. - -### 2. Brand Anchor Surface - -The top area contains the logo and brand title. On login and register, this remains the visual identity anchor. On reset password, the title becomes more task-driven, but the page still inherits the same spatial language and product mood. - -### 3. Primary Floating Card - -The form lives in a single, elevated primary card with softened corners, restrained shadows, and calm separation from the background. This card should feel intentional and product-grade rather than like a white panel. - -### 4. Secondary Assistive Layer - -Links, helper text, step hints, resend actions, and supplemental explanations belong to a lower-emphasis support layer. These elements should feel connected to the primary card without visually competing with it. - -## Shared Auth Composition - -All three auth screens should use a unified composition pattern. - -- Top brand or task heading area -- Main floating card for the active task -- Internal grouped sections inside the card -- Lightweight transition area for secondary actions - -This creates cross-screen consistency while allowing each flow to have different emphasis. - -## Screen Designs - -### Login Screen - -The login screen should be the most restrained and focused of the three. - -Structure: - -- logo and brand title at top -- one compact floating form card -- email field -- password field -- primary login CTA -- low-emphasis forgot-password action inside the card -- lightweight bottom switch action to register - -Intent: - -- reduce visual noise -- make the login action feel confident and central -- keep account switching and recovery available without stealing focus - -### Register Screen - -The register screen should feel richer than login, but still composed. - -Structure: - -- same brand anchor as login -- taller primary card -- grouped section for core account information -- grouped section for invite code as an optional enhancement -- subtle progress indicator treated as part of the card rhythm, not as a crude progress bar -- primary CTA for moving to verification -- lightweight switch action to login - -Invite code treatment: - -- remains a fixed-length segmented input -- visually grouped as an optional section, not mixed into the main required inputs -- supported by a nearby low-emphasis explanation block - -### Reset Password Screen - -The reset-password screen should shift from a flat all-fields form into a guided two-stage flow. - -Structure: - -- task heading at top -- primary floating card -- stage one: email input plus send-code action -- after successful code send, reveal stage two inside the same card -- stage two: segmented verification code input, resend action, new password, confirm password, submit CTA -- lightweight return-to-login switch action below - -Intent: - -- create procedural clarity -- avoid forcing all fields onto the screen at once -- make the verification code area feel deliberate and product-grade - -## Fixed-Length Segmented Input Design - -The user explicitly prefers fixed-length segmented input for both verification code and invite code. This input pattern will be preserved. - -However, it will be redesigned to feel like a premium grouped input rather than a row of raw boxes. - -Design principles: - -- the individual cells must read as one continuous input group -- current focus should be visible at both cell level and group level -- spacing should be balanced and calm -- filled state should feel confident and readable -- disabled and error states should be obvious without becoming harsh - -Required states: - -- default -- focused group -- active cell -- filled -- error -- disabled - -Usage rules: - -- verification code remains segmented -- invite code remains segmented -- resend action is no longer visually fused to the segmented input row - -## Button Design - -The CTA button must stop reading as a flat blue block. - -New button intent: - -- deeper and calmer brand blue -- stronger tactile weight -- premium capsule shape -- light depth through tonal layering and restrained shadow -- pressed and disabled states that clearly change material weight - -Button hierarchy: - -- primary button: for submission and key forward progress -- secondary button: for bounded alternative actions -- text-link action: for lightweight transitions and low-risk actions - -Color strategy: - -- blue remains the anchor, but is used with restraint -- the strongest emphasis goes to the main CTA only -- supporting actions should not look equally loud - -## Input Field Design - -Text fields should move from generic bordered rectangles to soft embedded surfaces. - -Desired qualities: - -- calmer default appearance -- stronger focus clarity -- reduced raw border noise -- consistent radius and spacing rhythm -- visually integrated label, input, and helper message states - -## Feedback System Redesign - -The feedback system is split into two coordinated layers. - -### 1. Global Floating Toast - -Global toast is a lightweight floating feedback card presented in the safe area near the top of the screen. - -Use global toast for: - -- cross-component success feedback -- async result notifications -- cross-step flow results -- errors that are not tied to a single visible field - -Do not use global toast for: - -- simple inline validation -- field-level format guidance -- contextual explanations that belong near the active input area - -Visual direction: - -- floating product card, not a system notification strip -- soft surface and rounded shape -- restrained state tinting -- icon plus title/message hierarchy if needed -- gentle slide/fade motion - -### 2. In-Component Contextual Message - -Contextual messages live inside the component or form group they explain. - -Use contextual messages for: - -- validation near fields -- warnings tied to current form state -- helper guidance for invite code or verification flow -- inline explanation of what the user should fix next - -Visual direction: - -- integrated into the card hierarchy -- lighter than a toast -- close to the related field or group -- consistent state styling across info, warning, and error - -### Responsibility Boundary - -- global toast = result-oriented, temporary, cross-context feedback -- inline message = contextual, explanatory, local feedback - -This separation prevents toast overuse and makes forms feel calmer. - -## Motion Language - -Motion should be soft and minimal. - -Use motion for: - -- toast entrance and dismissal -- reset-password stage reveal -- button press feedback -- segmented input focus continuity - -Avoid: - -- springy or playful motion -- overlapping dramatic transitions -- flashy state animations - -## Shared Component Impact - -The redesign likely requires updates or additions to shared widgets rather than one-off page styling. - -Expected shared component work: - -- refine `AppButton` -- refine or replace current toast visuals -- refine `AppBanner` or introduce a shared inline message presentation built on the same semantics -- redesign `FixedLengthCodeInput` -- add a reusable auth surface wrapper if needed - -## Verification Strategy - -Because this work is UI-heavy, verification should focus on correctness, consistency, and safe reuse. - -Primary verification targets: - -- `flutter analyze` -- impacted auth tests -- targeted widget tests only where reusable interactive widgets become materially more complex -- manual visual review of the three auth screens and feedback states - -## Success Criteria - -The redesign is successful when all of the following are true: - -- the three auth screens feel like one coherent product system -- the UI no longer resembles a plain white form page with blue buttons -- the main CTA has better perceived material quality -- reset password has a clearer and more attractive two-stage flow -- segmented code input remains intact but feels premium -- global toasts and inline messages have clear responsibility boundaries -- the feedback system feels native to the product rather than bolted on -- the result plausibly matches the visual language standard for a polished assistant app diff --git a/docs/plans/2026-03-13-auth-pages-redesign-implementation-plan.md b/docs/plans/2026-03-13-auth-pages-redesign-implementation-plan.md deleted file mode 100644 index e85e179..0000000 --- a/docs/plans/2026-03-13-auth-pages-redesign-implementation-plan.md +++ /dev/null @@ -1,337 +0,0 @@ -# Auth Pages Redesign Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Redesign the Flutter auth pages and feedback system so login, register, reset-password, segmented code input, and toast/inline feedback all match the project visual design language while preserving existing logic. - -**Architecture:** Keep auth business logic, cubits, navigation, and repository behavior unchanged. Move the redesign into shared design tokens and shared widgets first, then rebuild each auth screen on top of those primitives so the result is reusable and consistent. - -**Tech Stack:** Flutter, flutter_bloc, go_router, existing design tokens and shared widgets under `apps/lib/` - ---- - -### Task 1: Audit reusable auth and feedback primitives - -**Files:** -- Modify: `apps/lib/core/theme/design_tokens.dart` -- Modify: `apps/lib/shared/widgets/app_button.dart` -- Modify: `apps/lib/shared/widgets/fixed_length_code_input.dart` -- Modify: `apps/lib/shared/widgets/toast/toast.dart` -- Modify: `apps/lib/shared/widgets/banner/app_banner.dart` -- Test: `apps/test/` relevant existing widget or feature tests if impacted - -**Step 1: Identify missing token semantics before UI changes** - -Review current colors, spacing, radius, and surface semantics. List any missing token roles needed for: -- atmospheric auth background -- floating card border/shadow layering -- premium CTA surface states -- segmented input states -- toast and inline message states - -**Step 2: Add only the minimal new tokens needed** - -Update `apps/lib/core/theme/design_tokens.dart` with shared semantic tokens rather than page-local constants. - -Expected areas: -- auth background surface tones -- auth card border/highlight tones -- stronger button tonal roles -- inline message backgrounds/borders/text roles -- toast surface roles if existing status tokens are insufficient - -**Step 3: Run a quick compile-oriented check mentally against all planned components** - -Confirm the new tokens are generic enough for shared reuse and not named after individual screens. - -**Step 4: Commit checkpoint note** - -Do not create a git commit unless explicitly requested by the user. - -### Task 2: Redesign shared CTA and link interaction surfaces - -**Files:** -- Modify: `apps/lib/shared/widgets/app_button.dart` -- Modify: `apps/lib/shared/widgets/link_button.dart` -- Test: existing auth tests if button usage affects behavior - -**Step 1: Write the failing test if widget behavior changes materially** - -If the redesign introduces new behavior beyond styling, add a targeted widget test. If changes remain visual-only, document that no new test is added per lightweight UI testing policy. - -**Step 2: Refactor `AppButton` into a stronger material hierarchy** - -Implement: -- calmer premium CTA surface -- better disabled state separation -- consistent capsule-like shape -- optional secondary/outlined appearance if already used - -Keep public API stable unless a small safe extension is clearly needed. - -**Step 3: Refine `LinkButton` hit area and visual tone** - -Implement: -- clearer touch target -- lighter emphasis than CTA -- better pressed and disabled feel - -**Step 4: Run impacted checks** - -Run: `flutter analyze` - -Expected: no new analyzer issues from button or link widget changes. - -### Task 3: Redesign segmented input as a premium grouped control - -**Files:** -- Modify: `apps/lib/shared/widgets/fixed_length_code_input.dart` -- Test: add or update a widget test only if interaction logic changes materially - -**Step 1: Write the failing test for any changed interaction behavior** - -If focus progression, formatting, or semantic behavior changes, add a widget test that captures the intended interaction. If only visuals change, document why no new test is added. - -**Step 2: Rebuild the visual structure of the segmented input** - -Implement: -- clearer grouped container feel -- balanced cell rhythm -- group-level focus cue plus active-cell cue -- stronger filled state -- polished error and disabled states - -**Step 3: Preserve existing logical behavior** - -Keep: -- fixed length handling -- allowed character filtering -- uppercase support -- autofill compatibility - -**Step 4: Run focused verification** - -Run: `flutter analyze` - -Expected: no new issues from segmented input refactor. - -### Task 4: Rebuild global toast and inline message visuals - -**Files:** -- Modify: `apps/lib/shared/widgets/toast/toast.dart` -- Modify: `apps/lib/shared/widgets/banner/app_banner.dart` -- Modify: `apps/lib/shared/widgets/toast/toast_type.dart` if needed -- Modify: `apps/lib/shared/widgets/toast/toast_type_config.dart` -- Test: add widget tests only if feedback behavior semantics change - -**Step 1: Inspect current toast config implementation** - -Review `toast_type_config.dart` and align the redesign with shared semantic tokens. - -**Step 2: Redesign global toast as a floating product card** - -Implement: -- safe-area aware floating card -- refined tint, border, icon, and text hierarchy -- restrained shadow and motion -- stable dismissal behavior - -**Step 3: Redesign inline message presentation** - -Implement: -- lighter component-level message styling -- consistent relationship to current form group -- clear differentiation from toast while sharing status semantics - -**Step 4: Preserve system rules** - -Keep: -- `Toast.show(...)` for transient global feedback -- `AppBanner` for persistent inline feedback -- no raw `ScaffoldMessenger` - -**Step 5: Run focused verification** - -Run: `flutter analyze` - -Expected: no analyzer issues. - -### Task 5: Add a reusable auth surface composition primitive - -**Files:** -- Modify: `apps/lib/features/auth/ui/widgets/auth_page_scaffold.dart` -- Create or Modify: `apps/lib/features/auth/ui/widgets/` shared auth surface widgets as needed - -**Step 1: Evaluate whether the existing scaffold can express the new surface hierarchy** - -Check whether `AuthPageScaffold` can support: -- atmospheric background treatment -- safe-area balanced centering -- top brand anchor spacing -- floating card composition - -**Step 2: Implement the minimal reusable auth layout primitives** - -Possible additions: -- auth hero header -- auth floating card shell -- grouped section wrapper - -Only create what is reused by at least two auth screens. - -**Step 3: Keep layout semantics explicit** - -Ensure every `Row` and `Column` has explicit `crossAxisAlignment` and layout intent remains traceable. - -### Task 6: Rebuild the login page UI on top of the shared primitives - -**Files:** -- Modify: `apps/lib/features/auth/ui/screens/login_screen.dart` - -**Step 1: Preserve all existing logic paths** - -Do not change: -- cubit usage -- submit flow -- auth bloc event dispatch -- navigation targets - -**Step 2: Replace the existing layout with the new design** - -Implement: -- brand anchor header -- compact floating login card -- refined email and password groups -- CTA-first hierarchy -- lightweight forgot-password action -- low-emphasis register switch - -**Step 3: Reconnect inline feedback to the new grouping** - -Place validation and error banners where they support the form rhythm rather than interrupt it. - -**Step 4: Run impacted checks** - -Run: `flutter analyze` - -Expected: login screen compiles cleanly. - -### Task 7: Rebuild the register page UI with grouped invite code treatment - -**Files:** -- Modify: `apps/lib/features/auth/ui/screens/register_screen.dart` - -**Step 1: Preserve all existing logic paths** - -Do not change: -- cubit usage -- invite code normalization -- validation behavior -- route transition to verification page -- silent code send behavior - -**Step 2: Rebuild required and optional sections** - -Implement: -- shared brand anchor header -- taller floating card -- grouped account info section -- separate optional invite code section -- refined progress indicator treatment -- improved footer switch action - -**Step 3: Keep segmented invite code input** - -Use the redesigned shared segmented input without changing its fixed-length semantics. - -**Step 4: Run impacted checks** - -Run: `flutter analyze` - -Expected: register screen compiles cleanly. - -### Task 8: Rebuild the reset-password page as a guided two-stage flow - -**Files:** -- Modify: `apps/lib/features/auth/ui/screens/reset_password_screen.dart` - -**Step 1: Preserve all existing logic and feedback semantics** - -Do not change: -- cubit interactions -- send code and resend behavior -- submit behavior -- success redirect -- toast semantics - -**Step 2: Recompose the screen into staged groups** - -Implement: -- task-focused header -- stage one email + send code group -- stage two reveal for code and password reset controls after send success -- separate resend action from the code row while preserving usability - -**Step 3: Reconnect segmented verification code input and inline messaging** - -Make the code group feel premium and central without overcrowding the card. - -**Step 4: Run impacted checks** - -Run: `flutter analyze` - -Expected: reset-password screen compiles cleanly. - -### Task 9: Run verification and inspect affected tests - -**Files:** -- Test: impacted auth tests and any new widget tests added - -**Step 1: Run analyzer** - -Run: `flutter analyze` - -Expected: PASS with no new issues. - -**Step 2: Run impacted auth tests** - -Run an auth-focused test subset appropriate to the changed files, for example: - -```bash -flutter test apps/test/features/auth -``` - -Adjust the exact command to the repository's Flutter test layout if needed. - -Expected: existing auth tests remain green; any new targeted widget tests pass. - -**Step 3: Manual visual review checklist** - -Verify: -- login looks calm and premium -- register invite code section feels optional but intentional -- reset-password stage flow is clear -- segmented code input feels grouped and polished -- toast feels like a product surface -- inline messages feel local and non-intrusive - -### Task 10: Final review and handoff - -**Files:** -- Modify: any touched files from previous tasks if final polish is needed - -**Step 1: Check for consistency drift** - -Ensure the three auth pages, toast, inline messages, buttons, and segmented input all feel like one system. - -**Step 2: Confirm no scope creep changed logic unexpectedly** - -Re-check that routing, auth behavior, and validation rules remain intact. - -**Step 3: Prepare concise handoff summary** - -Include: -- files changed -- verification commands run -- test results -- any follow-up visual refinements still worth considering diff --git a/docs/plans/2026-03-13-home-screen-visual-refresh.md b/docs/plans/2026-03-13-home-screen-visual-refresh.md deleted file mode 100644 index f5c83ab..0000000 --- a/docs/plans/2026-03-13-home-screen-visual-refresh.md +++ /dev/null @@ -1,484 +0,0 @@ -# Home Screen Visual Refresh Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Rebuild the post-login home screen into a calm, premium assistant homepage with a layered background field, a clearer conversation stage, and a floating input island that carries text, voice, and attachment states without relying on helper copy. - -**Architecture:** Keep the existing chat data flow and AG-UI behavior unchanged. Refactor the screen into a surface-based composition: background field at the bottom, content layer for header and conversation stage in the middle, and a floating input layer on top. Move visual semantics into shared design tokens first, then rebuild `MessageComposer` and `HomeScreen` around those tokens. - -**Tech Stack:** Flutter, Material, flutter_bloc, existing design token system, existing widget tests in `apps/test/features/home/ui/widgets/`. - ---- - -### Task 1: Add home surface tokens - -**Files:** -- Modify: `apps/lib/core/theme/design_tokens.dart` -- Reference: `apps/rules/visual_design_language.md` - -**Step 1: Write the failing test** - -Add a widget test assertion that depends on a new home token being used by `MessageComposer`, for example: - -```dart -testWidgets('composer uses refreshed home surface token', (tester) async { - await tester.pumpWidget(_buildTestApp( - mode: MessageComposerMode.text, - process: MessageComposerProcess.idle, - hasMessage: false, - isWaitingAgent: false, - )); - - final container = tester.widget( - find.byKey(messageComposerContainerKey), - ); - final decoration = container.decoration! as BoxDecoration; - expect(decoration.color, AppColors.homeComposerShell); -}); -``` - -**Step 2: Run test to verify it fails** - -Run: `flutter test apps/test/features/home/ui/widgets/home_composer_test.dart` - -Expected: FAIL because `homeComposerShell` does not exist yet. - -**Step 3: Write minimal implementation** - -Add the first batch of home-specific tokens in `apps/lib/core/theme/design_tokens.dart`, keeping names semantic instead of layout-specific: - -```dart -static const homeBackgroundTop = Color(0xFFF5F9FF); -static const homeBackgroundBottom = Color(0xFFF7FAFE); -static const homeBackgroundGlow = Color(0xFFDCEBFF); -static const homeBackgroundGlowSoft = Color(0xFFF1F6FF); -static const homeToolbarSurface = Color(0xF2FFFFFF); -static const homeToolbarBorder = Color(0xD9E6F7); -static const homeConversationSurface = Color(0xBFFFFFFF); -static const homeConversationBorder = Color(0xDDE8F6); -static const homeComposerShell = Color(0xFDFCFEFF); -static const homeComposerInner = Color(0xFFF7FAFE); -static const homeComposerBorder = Color(0xD7E3F3); -static const homeComposerAccent = Color(0xFFEAF3FF); -static const homeAttachmentSurface = Color(0xFFF3F7FD); -``` - -**Step 4: Run test to verify it passes** - -Run: `flutter test apps/test/features/home/ui/widgets/home_composer_test.dart` - -Expected: PASS, with the new token available to downstream widgets. - -**Step 5: Commit** - -```bash -git add apps/lib/core/theme/design_tokens.dart apps/test/features/home/ui/widgets/home_composer_test.dart -git commit -m "feat: add home screen surface tokens" -``` - -### Task 2: Extract background field and floating header primitives - -**Files:** -- Create: `apps/lib/features/home/ui/widgets/home_background_field.dart` -- Create: `apps/lib/features/home/ui/widgets/home_floating_header.dart` -- Modify: `apps/lib/features/home/ui/screens/home_screen.dart` - -**Step 1: Write the failing test** - -Create a focused widget test for the new background field composition: - -```dart -testWidgets('home background field renders layered glow surfaces', (tester) async { - await tester.pumpWidget(const MaterialApp( - home: Scaffold(body: HomeBackgroundField()), - )); - - expect(find.byKey(homeBackgroundFieldKey), findsOneWidget); - expect(find.byKey(homeTopGlowKey), findsOneWidget); - expect(find.byKey(homeBottomGlowKey), findsOneWidget); -}); -``` - -**Step 2: Run test to verify it fails** - -Run: `flutter test apps/test/features/home/ui/widgets/home_background_field_test.dart` - -Expected: FAIL because the widget file and keys do not exist. - -**Step 3: Write minimal implementation** - -Build a reusable background widget using only tokens and soft gradients: - -```dart -class HomeBackgroundField extends StatelessWidget { - const HomeBackgroundField({super.key}); - - @override - Widget build(BuildContext context) { - return DecoratedBox( - key: homeBackgroundFieldKey, - decoration: const BoxDecoration( - gradient: LinearGradient( - begin: Alignment.topCenter, - end: Alignment.bottomCenter, - colors: [ - AppColors.homeBackgroundTop, - AppColors.homeBackgroundBottom, - ], - ), - ), - child: Stack( - children: const [ - _TopGlow(), - _BottomGlow(), - ], - ), - ); - } -} -``` - -Add a matching lightweight floating header widget so `HomeScreen` stops inlining all visual treatment in one file. - -**Step 4: Run test to verify it passes** - -Run: `flutter test apps/test/features/home/ui/widgets/home_background_field_test.dart` - -Expected: PASS. - -**Step 5: Commit** - -```bash -git add apps/lib/features/home/ui/widgets/home_background_field.dart apps/lib/features/home/ui/widgets/home_floating_header.dart apps/lib/features/home/ui/screens/home_screen.dart apps/test/features/home/ui/widgets/home_background_field_test.dart -git commit -m "feat: add layered home screen background primitives" -``` - -### Task 3: Rebuild `MessageComposer` as a floating input island - -**Files:** -- Modify: `apps/lib/shared/widgets/message_composer.dart` -- Modify: `apps/test/features/home/ui/widgets/home_composer_test.dart` - -**Step 1: Write the failing test** - -Extend the existing composer tests to enforce the new structure: - -```dart -testWidgets('composer exposes shell and inner surface', (tester) async { - await tester.pumpWidget(_buildTestApp( - mode: MessageComposerMode.text, - process: MessageComposerProcess.idle, - hasMessage: false, - isWaitingAgent: false, - )); - - expect(find.byKey(messageComposerShellKey), findsOneWidget); - expect(find.byKey(messageComposerInnerKey), findsOneWidget); -}); -``` - -Add another test to ensure the hold-to-speak state stays within the same shell: - -```dart -testWidgets('recording state keeps unified floating shell', (tester) async { - await tester.pumpWidget(_buildTestApp( - mode: MessageComposerMode.holdToSpeak, - process: MessageComposerProcess.recording, - hasMessage: false, - isWaitingAgent: false, - )); - - expect(find.byKey(messageComposerShellKey), findsOneWidget); - expect(find.text('松开发送'), findsOneWidget); -}); -``` - -**Step 2: Run test to verify it fails** - -Run: `flutter test apps/test/features/home/ui/widgets/home_composer_test.dart` - -Expected: FAIL because the new structure keys do not exist. - -**Step 3: Write minimal implementation** - -Refactor `MessageComposer` from a single decorated `Container` into a layered shell: - -```dart -return Container( - key: messageComposerShellKey, - padding: const EdgeInsets.all(AppSpacing.sm), - decoration: BoxDecoration( - color: AppColors.homeComposerShell, - borderRadius: BorderRadius.circular(AppRadius.xxl), - border: Border.all(color: AppColors.homeComposerBorder), - boxShadow: const [ - BoxShadow(...), - ], - ), - child: Container( - key: messageComposerInnerKey, - decoration: BoxDecoration( - color: AppColors.homeComposerInner, - borderRadius: BorderRadius.circular(AppRadius.xl), - ), - child: Row(...), - ), -); -``` - -Keep all current callbacks and AG-UI-adjacent behavior intact. Do not change event names or chat flow. - -**Step 4: Run test to verify it passes** - -Run: `flutter test apps/test/features/home/ui/widgets/home_composer_test.dart` - -Expected: PASS, including the existing callback and state-priority assertions. - -**Step 5: Commit** - -```bash -git add apps/lib/shared/widgets/message_composer.dart apps/test/features/home/ui/widgets/home_composer_test.dart -git commit -m "feat: rebuild composer as floating input island" -``` - -### Task 4: Integrate attachment strip into the input island stack - -**Files:** -- Create: `apps/lib/features/home/ui/widgets/home_attachment_strip.dart` -- Modify: `apps/lib/features/home/ui/screens/home_screen.dart` - -**Step 1: Write the failing test** - -Add a widget test that verifies selected images render in a unified strip above the composer shell: - -```dart -testWidgets('selected images render in attachment strip above composer', (tester) async { - // Pump HomeScreen with seeded selected images via a test-only constructor hook. - expect(find.byKey(homeAttachmentStripKey), findsOneWidget); -}); -``` - -**Step 2: Run test to verify it fails** - -Run: `flutter test apps/test/features/home/ui/widgets/home_screen_input_stack_test.dart` - -Expected: FAIL because the strip widget and key do not exist. - -**Step 3: Write minimal implementation** - -Create a dedicated strip widget that uses the same home surface language: - -```dart -class HomeAttachmentStrip extends StatelessWidget { - const HomeAttachmentStrip({ - super.key, - required this.images, - required this.onRemove, - }); - - final List images; - final ValueChanged onRemove; - - @override - Widget build(BuildContext context) { - if (images.isEmpty) return const SizedBox.shrink(); - return Container( - key: homeAttachmentStripKey, - padding: const EdgeInsets.all(AppSpacing.sm), - decoration: BoxDecoration( - color: AppColors.homeAttachmentSurface, - borderRadius: BorderRadius.circular(AppRadius.xl), - ), - child: Wrap(...), - ); - } -} -``` - -Mount it in the same bottom stack as the composer, not in the main scroll column. - -**Step 4: Run test to verify it passes** - -Run: `flutter test apps/test/features/home/ui/widgets/home_screen_input_stack_test.dart` - -Expected: PASS. - -**Step 5: Commit** - -```bash -git add apps/lib/features/home/ui/widgets/home_attachment_strip.dart apps/lib/features/home/ui/screens/home_screen.dart apps/test/features/home/ui/widgets/home_screen_input_stack_test.dart -git commit -m "feat: unify home attachments with composer stack" -``` - -### Task 5: Recompose `HomeScreen` around stage + floating bottom stack - -**Files:** -- Modify: `apps/lib/features/home/ui/screens/home_screen.dart` -- Reference: `apps/lib/features/home/ui/screens/home_sheet.dart` - -**Step 1: Write the failing test** - -Add a focused home screen layout test: - -```dart -testWidgets('home screen shows floating header, conversation stage, and bottom input stack', (tester) async { - await tester.pumpWidget(buildHomeScreenForTest()); - - expect(find.byKey(homeFloatingHeaderKey), findsOneWidget); - expect(find.byKey(homeConversationStageKey), findsOneWidget); - expect(find.byKey(homeBottomInputStackKey), findsOneWidget); -}); -``` - -**Step 2: Run test to verify it fails** - -Run: `flutter test apps/test/features/home/ui/widgets/home_screen_layout_test.dart` - -Expected: FAIL because the layout keys are not present. - -**Step 3: Write minimal implementation** - -Refactor the page into a single `Stack` with explicit layers: - -```dart -return Scaffold( - body: SafeArea( - child: Stack( - children: [ - const Positioned.fill(child: HomeBackgroundField()), - Column( - crossAxisAlignment: CrossAxisAlignment.stretch, - children: [ - const HomeFloatingHeader(), - Expanded(child: _buildConversationStage(context, state)), - ], - ), - _buildBottomInputStack(context, state), - if (_isRecording) _buildRecordingGestureOverlay(), - ], - ), - ), -); -``` - -Inside `_buildConversationStage`, keep existing history/message rendering logic but place it inside a calmer stage container with stable bottom padding so the floating composer never overlaps message content. - -**Step 4: Run test to verify it passes** - -Run: `flutter test apps/test/features/home/ui/widgets/home_screen_layout_test.dart` - -Expected: PASS. - -**Step 5: Commit** - -```bash -git add apps/lib/features/home/ui/screens/home_screen.dart apps/test/features/home/ui/widgets/home_screen_layout_test.dart -git commit -m "feat: recompose home screen into layered assistant stage" -``` - -### Task 6: Tune empty-state and waiting-state presentation without adding helper copy - -**Files:** -- Modify: `apps/lib/features/home/ui/screens/home_screen.dart` - -**Step 1: Write the failing test** - -Add a test that verifies the empty state uses a dedicated stage surface instead of only centered text: - -```dart -testWidgets('empty state renders stage surface without relying on helper copy', (tester) async { - await tester.pumpWidget(buildEmptyHomeScreenForTest()); - expect(find.byKey(homeConversationStageKey), findsOneWidget); - expect(find.byKey(homeEmptyStateOrbKey), findsOneWidget); -}); -``` - -**Step 2: Run test to verify it fails** - -Run: `flutter test apps/test/features/home/ui/widgets/home_screen_layout_test.dart` - -Expected: FAIL because the empty-state focal surface does not exist. - -**Step 3: Write minimal implementation** - -Replace the current `Center(Text('开始对话吧'))` style empty state with a focal surface that uses shape, spacing, and a soft orb layer instead of explanatory copy: - -```dart -Widget _buildEmptyConversationStage() { - return Center( - child: Container( - key: homeEmptyStateOrbKey, - width: 220, - height: 220, - decoration: BoxDecoration( - shape: BoxShape.circle, - gradient: RadialGradient(...), - ), - ), - ); -} -``` - -Waiting state should sit at the lower edge of the stage, visually connected to the composer instead of appearing as a detached loading row. - -**Step 4: Run test to verify it passes** - -Run: `flutter test apps/test/features/home/ui/widgets/home_screen_layout_test.dart` - -Expected: PASS. - -**Step 5: Commit** - -```bash -git add apps/lib/features/home/ui/screens/home_screen.dart apps/test/features/home/ui/widgets/home_screen_layout_test.dart -git commit -m "feat: refine home empty and waiting states" -``` - -### Task 7: Verify visual refresh and document manual QA - -**Files:** -- Modify: `docs/plans/2026-03-13-home-screen-visual-refresh.md` - -**Step 1: Run automated verification** - -Run: - -```bash -flutter test apps/test/features/home/ui/widgets/home_composer_test.dart apps/test/features/home/ui/widgets/home_background_field_test.dart apps/test/features/home/ui/widgets/home_screen_layout_test.dart apps/test/features/home/ui/widgets/home_screen_input_stack_test.dart -``` - -Expected: PASS. - -**Step 2: Run static verification** - -Run: - -```bash -dart format apps/lib/core/theme/design_tokens.dart apps/lib/features/home/ui/screens/home_screen.dart apps/lib/features/home/ui/widgets/home_background_field.dart apps/lib/features/home/ui/widgets/home_floating_header.dart apps/lib/features/home/ui/widgets/home_attachment_strip.dart apps/lib/shared/widgets/message_composer.dart apps/test/features/home/ui/widgets/home_composer_test.dart apps/test/features/home/ui/widgets/home_background_field_test.dart apps/test/features/home/ui/widgets/home_screen_layout_test.dart apps/test/features/home/ui/widgets/home_screen_input_stack_test.dart -flutter analyze -``` - -Expected: PASS. - -**Step 3: Run manual QA** - -Verify on a phone-sized simulator or device: - -```text -1. 首页空态 first impression 不依赖提示文案，仍然有清晰主场感 -2. 顶部浮层不抢焦点，底部输入岛是最稳定视觉锚点 -3. 文本/语音/转写/等待态切换时，输入岛壳体保持连续 -4. 附件预览与输入区属于同一层级，不再像临时插块 -5. 消息列表滚动到底部时，不会被悬浮输入岛遮挡 -``` - -**Step 4: Update plan status note** - -Append a short verification note to this plan with pass/fail status and any follow-up token work. - -**Step 5: Commit** - -```bash -git add docs/plans/2026-03-13-home-screen-visual-refresh.md -git commit -m "docs: record home screen visual refresh verification" -``` diff --git a/docs/protocols/agent-chat-messages.md b/docs/protocols/agent-chat-messages.md deleted file mode 100644 index dfe62f0..0000000 --- a/docs/protocols/agent-chat-messages.md +++ /dev/null @@ -1,220 +0,0 @@ -# Agent Chat Messages Protocol - -> **NOTE**: This document is the single source of truth. All implementations must follow this specification. - -## Overview - -Chat messages in agent conversations with metadata for tracking and telemetry. - -## Version - -- **Current**: `1.0` -- **Status**: Active - ---- - -## Message Metadata - -```typescript -interface AgentChatMessageMetadata { - run_id?: string; // Unique run identifier - stage?: string; // Processing stage (e.g., "intent", "execution") - latency_ms?: number; // Processing latency in milliseconds - message_id?: string; // Message identifier - [key: string]: any; // Additional custom fields allowed -} -``` - ---- - -## Database Field - -| Field | Type | Description | -|-------|------|-------------| -| metadata | jsonb | Message metadata including run_id, stage, latency_ms, message_id | - ---- - -## JSON Examples - -### Basic Message Metadata - -```json -{ - "run_id": "run_1773286460762" -} -``` - -### Stage Completion Metadata - -```json -{ - "run_id": "run_1773286460762", - "stage": "intent", - "latency_ms": 2610, - "message_id": "intent-run_1773286460762" -} -``` - -### Tool Execution Metadata - -```json -{ - "run_id": "run_1773287162123", - "stage": "tool_execution", - "latency_ms": 1500, - "message_id": "tool_run_abc123", - "tool_name": "calendar_create_event" -} -``` - ---- - -## User Message Attachments - -### Overview - -When a user sends a message with binary attachments (e.g., images), the frontend uploads the file to storage first, then sends a signed URL to the backend. The backend parses the signed URL to extract storage metadata and persists it with the message. - -### Flow - -``` -Frontend Backend -──────────────────────────────────────────────────────────────── -1. Upload file - POST /api/v1/agent/attachments - ──────────────────────────────> - <────────────────────────────── - {bucket, path, mime_type, url: signed_url} - -2. Send message with binary block - POST /api/v1/agent/runs - content: [ - {type: "text", text: "..."}, - {type: "binary", mimeType: "image/jpeg", url: signed_url} - ] - ──────────────────────────────> - -3. Backend parses signed URL - parse_signed_url(url) → {bucket, path} - -4. Persist to database - metadata.user_message_attachments = {bucket, path, mime_type} - -5. Return history (GET /history) - <────────────────────────────── - messages: [{ - id: "msg-1", - seq: 1, - role: "user", - content: "...", - metadata: { - user_message_attachments: {bucket, path, mime_type} - }, - timestamp: "2026-03-13T10:00:00Z" - }] - -6. Resolve temporary URL for rendering - GET /api/v1/agent/attachments/signed-url?bucket=...&path=... - ──────────────────────────────> - <────────────────────────────── - {bucket, path, url} -``` - -### Signed URL Format - -Supabase signed URL format: -``` -https://{project}.supabase.co/storage/v1/object/sign/{bucket}/{path}?token={jwt} -``` - -Backend parses to extract: -- `bucket`: URL path segment after `/sign/` -- `path`: Remaining path after bucket - -### Metadata Schema - -```typescript -interface UserMessageAttachments { - bucket: string; // Storage bucket name - path: string; // Object storage path - mime_type: string; // MIME type (e.g., "image/jpeg") -} - -interface AgentChatMessageMetadata { - // ... existing fields - user_message_attachments?: UserMessageAttachments; -} -``` - -### Database Storage - -| Field | Type | Description | -|-------|------|-------------| -| metadata | jsonb | Contains user_message_attachments with bucket, path, mime_type | - -### Example - -**Request (POST /runs):** -```json -{ - "threadId": "thread-123", - "runId": "run-456", - "messages": [ - { - "id": "msg-1", - "role": "user", - "content": [ - {"type": "text", "text": "帮我看看这张图"}, - { - "type": "binary", - "mimeType": "image/jpeg", - "url": "https://xxx.supabase.co/storage/v1/object/sign/agent-files/agent-inputs/u/t/r/img.jpg?token=xxx" - } - ] - } - ] -} -``` - -**Stored Metadata:** -```json -{ - "user_message_attachments": { - "bucket": "agent-files", - "path": "agent-inputs/u/t/r/img.jpg", - "mime_type": "image/jpeg" - } -} -``` - -**History Response (GET /history):** -```json -{ - "messages": [ - { - "id": "msg-1", - "seq": 1, - "role": "user", - "content": "帮我看看这张图", - "metadata": { - "user_message_attachments": { - "bucket": "agent-files", - "path": "agent-inputs/u/t/r/img.jpg", - "mime_type": "image/jpeg" - } - }, - "timestamp": "2026-03-13T10:00:00Z" - } - ] -} -``` - -**Attachment URL Response (GET /attachments/signed-url):** -```json -{ - "bucket": "agent-files", - "path": "agent-inputs/u/t/r/img.jpg", - "url": "https://xxx.supabase.co/storage/v1/object/sign/agent-files/agent-inputs/u/t/r/img.jpg?token=yyy" -} -``` diff --git a/docs/protocols/agent-chat-sessions.md b/docs/protocols/agent-chat-sessions.md deleted file mode 100644 index efe5eb6..0000000 --- a/docs/protocols/agent-chat-sessions.md +++ /dev/null @@ -1,59 +0,0 @@ -# Agent Chat Sessions Protocol - -> **NOTE**: This document is the single source of truth. All implementations must follow this specification. - -## Overview - -Agent chat session state snapshot for preserving conversation context. - -## Version - -- **Current**: `1.0` -- **Status**: Active - ---- - -## Session State Snapshot - -```typescript -interface SessionStateSnapshot { - // Reserved for future use - // Currently unused, allowing custom extensions - [key: string]: any; -} -``` - ---- - -## Database Field - -| Field | Type | Description | -|-------|------|-------------| -| state_snapshot | jsonb | Session state for preserving conversation context | - ---- - -## JSON Examples - -### Empty State - -```json -{} -``` - -### Future Usage Example - -```json -{ - "conversation_context": { - "last_topic": "calendar_events", - "mentioned_dates": ["2026-03-15", "2026-03-20"] - }, - "agent_memory": { - "user_preferences": { - "timezone": "Asia/Shanghai", - "language": "zh-CN" - } - } -} -``` diff --git a/docs/protocols/agent/api-endpoints.md b/docs/protocols/agent/api-endpoints.md new file mode 100644 index 0000000..7cfbd8a --- /dev/null +++ b/docs/protocols/agent/api-endpoints.md @@ -0,0 +1,360 @@ +# Agent API Endpoints + +本文档列出所有 Agent 相关的 API 端点。 + +Base URL: `/api/v1/agent` + +--- + +## 端点清单 + +| 方法 | 路径 | 描述 | +|------|------|------| +| POST | `/runs` | 发起 Agent 运行 | +| GET | `/runs/{thread_id}/events` | SSE 事件流 | +| GET | `/history` | 获取对话历史快照 | +| POST | `/attachments` | 上传附件 | +| GET | `/attachments/signed-url` | 生成附件签名 URL | +| POST | `/transcribe` | 语音转文字 (ASR) | + +--- + +## 1. POST /runs + +发起一个 Agent 运行任务。 + +### Request + +Request Body: `RunAgentInput` + +详细数据结构见 [run-agent-input.md](./run-agent-input.md) + +### Response + +```typescript +{ + taskId: string, // 任务 ID + threadId: string, // 会话 ID + runId: string, // 运行 ID + created: string // ISO-8601 时间戳 +} +``` + +### Example + +```bash +curl -X POST https://api.example.com/api/v1/agent/runs \ + -H "Content-Type: application/json" \ + -d '{ + "threadId": "550e8400-e29b-41d4-a716-446655440000", + "runId": "run-001", + "state": {}, + "messages": [ + { + "id": "msg-001", + "role": "user", + "content": "帮我查一下北京今天的天气" + } + ], + "tools": [], + "context": [], + "forwardedProps": {} + }' +``` + +### Response Example + +```json +{ + "taskId": "task-abc123", + "threadId": "550e8400-e29b-41d4-a716-446655440000", + "runId": "run-001", + "created": "2026-03-16T10:00:00Z" +} +``` + +--- + +## 2. GET /runs/{thread_id}/events + +获取 SSE 事件流，用于实时接收 Agent 运行过程中的事件。 + +### Path Parameters + +| 参数 | 类型 | 描述 | +|------|------|------| +| thread_id | string | 会话 ID | + +### Query Parameters + +| 参数 | 类型 | 默认值 | 描述 | +|------|------|--------|------| +| Last-Event-ID | string | - | 可选，用于断点续传的事件 ID | +| idle_limit | integer | 300 | 最大空闲轮询次数 (1-3600) | + +### Headers + +| 参数 | 描述 | +|------|------| +| Last-Event-ID | 可选的事件 ID，用于从指定位置恢复 | + +### Response + +SSE (Server-Sent Events) 流，Content-Type: `text/event-stream` + +事件类型详情见 [sse-events.md](./sse-events.md) + +### Example + +```javascript +const eventSource = new EventSource( + 'https://api.example.com/api/v1/agent/runs/550e8400-e29b-41d4-a716-446655440000/events' +); + +eventSource.addEventListener('run.started', (e) => { + const data = JSON.parse(e.data); + console.log('Started:', data); +}); + +eventSource.addEventListener('text.delta', (e) => { + const data = JSON.parse(e.data); + console.log('Delta:', data.data.delta); +}); + +eventSource.addEventListener('run.finished', (e) => { + const data = JSON.parse(e.data); + console.log('Finished:', data); +}); +``` + +--- + +## 3. GET /history + +获取对话历史快照。 + +### Query Parameters + +| 参数 | 类型 | 必填 | 描述 | +|------|------|------|------| +| threadId | string | 否 | 会话 ID，不指定则返回最新会话 | +| before | date | 否 | 日期格式 `YYYY-MM-DD`，返回该日期之前的快照 | + +### Response + +```typescript +{ + scope: "history_day", + threadId: string | null, + day: string | null, // ISO date format "YYYY-MM-DD" + hasMore: boolean, + messages: HistoryMessage[] +} +``` + +详细数据结构见 [run-agent-input.md](./run-agent-input.md) + +### Example + +```bash +curl "https://api.example.com/api/v1/agent/history?threadId=550e8400-e29b-41d4-a716-446655440000&before=2026-03-15" +``` + +### Response Example + +```json +{ + "scope": "history_day", + "threadId": "550e8400-e29b-41d4-a716-446655440000", + "day": "2026-03-15", + "hasMore": false, + "messages": [ + { + "id": "msg-001", + "seq": 1, + "role": "user", + "content": "帮我创建一个日程", + "url": null, + "timestamp": "2026-03-15T10:00:00Z" + }, + { + "id": "msg-002", + "seq": 2, + "role": "assistant", + "content": "好的，我来帮您创建日程。", + "uiSchema": { + "version": "2.0", + "locale": "zh-CN", + "status": "success", + "theme": "default", + "root": { + "type": "stack", + "appearance": "card", + "children": [ + {"type": "text", "content": "日程已创建", "role": "title"}, + {"type": "badge", "label": "SUCCESS", "status": "success"} + ] + } + }, + "timestamp": "2026-03-15T10:00:05Z" + } + ] +} +``` + +--- + +## 4. POST /attachments + +上传附件到存储。 + +### Form Data + +| 参数 | 类型 | 描述 | +|------|------|------| +| threadId | string | 会话 ID | +| file | file | 要上传的文件 | + +### Response + +```typescript +{ + attachment: { + bucket: string, + path: string, + mimeType: string, + size: number, + url: string // 临时访问 URL + } +} +``` + +### Example + +```bash +curl -X POST https://api.example.com/api/v1/agent/attachments \ + -F "threadId=550e8400-e29b-41d4-a716-446655440000" \ + -F "file=@/path/to/image.png" +``` + +### Limits + +- 最大文件大小: 5MB +- 支持的文件类型: 见后端配置 + +--- + +## 5. GET /attachments/signed-url + +生成附件的签名 URL，用于直接访问存储中的文件。 + +### Query Parameters + +| 参数 | 类型 | 必填 | 描述 | +|------|------|------|------| +| bucket | string | 是 | 存储桶名称 | +| path | string | 是 | 文件路径 | + +### Response + +```typescript +{ + bucket: string, + path: string, + url: string // 签名 URL +} +``` + +### Example + +```bash +curl "https://api.example.com/api/v1/agent/attachments/signed-url?bucket=agent-inputs&path=user-123/image.png" +``` + +### Response Example + +```json +{ + "bucket": "agent-inputs", + "path": "user-123/image.png", + "url": "https://storage.example.com/agent-inputs/user-123/image.png?signature=abc123..." +} +``` + +--- + +## 6. POST /transcribe + +语音转文字 (ASR)。 + +### Request + +Form Data: + +| 参数 | 类型 | 描述 | +|------|------|------| +| audio | file | 音频文件 (WAV 格式) | + +### Headers + +| 参数 | 描述 | +|------|------| +| content-length | 文件大小 | + +### Response + +```typescript +{ + transcript: string, // 转录文本 + language: string, // 检测到的语言 + duration: number // 音频时长 (秒) +} +``` + +### Example + +```bash +curl -X POST https://api.example.com/api/v1/agent/transcribe \ + -F "audio=@/path/to/audio.wav" +``` + +### Response Example + +```json +{ + "transcript": "今天天气真不错", + "language": "zh-CN", + "duration": 3.5 +} +``` + +### Limits + +- 支持格式: `audio/wav`, `audio/x-wav`, `audio/wave` +- 最大文件大小: 10MB +- 速率限制: 20 次/分钟/用户 + +--- + +## 错误响应 + +所有端点可能返回以下错误： + +| 状态码 | 描述 | +|--------|------| +| 400 | 请求参数错误 | +| 401 | 未认证 | +| 403 | 无权限 | +| 404 | 资源不存在 | +| 413 | 请求体过大 | +| 422 | 数据验证失败 | +| 429 | 速率限制 | +| 500 | 服务器内部错误 | + +### Error Response Format + +```json +{ + "detail": "错误详情信息" +} +``` diff --git a/docs/protocols/agent/run-agent-input.md b/docs/protocols/agent/run-agent-input.md new file mode 100644 index 0000000..6f7ec33 --- /dev/null +++ b/docs/protocols/agent/run-agent-input.md @@ -0,0 +1,449 @@ +# Agent Run Input Protocol + +> **NOTE**: This document defines the RunAgentInput data structure for `POST /api/v1/agent/runs`. + +## Version + +- **Current**: `1.0` +- **Status**: Active + +--- + +## Overview + +`POST /api/v1/agent/runs` accepts `RunAgentInput` as request body to initiate an agent execution. + +--- + +## RunAgentInput Schema + +```typescript +interface RunAgentInput { + threadId: string; // 必须是有效 UUID + runId: string; // 最大 128 字符 + parentRunId?: string; + state?: any; + messages: Message[]; // 最多 200 条 + tools?: Tool[]; + context?: Context[]; + forwardedProps?: any; +} +``` + +### Required Fields + +| Field | Type | Constraints | +|-------|------|-------------| +| `threadId` | string | 必须为有效 UUID | +| `runId` | string | 最大 128 字符 | +| `messages` | array | 最多 200 条，**必须恰好包含 1 条 user message** | +| `state` | any | - | +| `tools` | array | 可选 | +| `context` | array | 可选 | +| `forwardedProps` | any | - | + +--- + +## Message Types + +### UserMessage + +```typescript +interface UserMessage { + id: string; + role: "user"; + content: string | ContentBlock[]; + name?: string; + encryptedValue?: string; +} + +type ContentBlock = TextContentBlock | BinaryContentBlock; + +interface TextContentBlock { + type: "text"; + text: string; +} + +interface BinaryContentBlock { + type: "binary"; + mimeType: string; // 必须为 image/* 类型 + id?: string; + url?: string; // 必须是有效的 signed URL + data?: string; // 不允许使用 data 字段 + filename?: string; +} +``` + +### AssistantMessage + +```typescript +interface AssistantMessage { + id: string; + role: "assistant"; + content?: string | null; + name?: string | null; + toolCalls?: ToolCall[]; + encryptedValue?: string | null; +} + +interface ToolCall { + id: string; + type: "function"; + function: { + name: string; + arguments: string; + }; + encryptedValue?: string | null; +} +``` + +### SystemMessage + +```typescript +interface SystemMessage { + id: string; + role: "system"; + content: string; + name?: string | null; + encryptedValue?: string | null; +} +``` + +### ToolMessage + +```typescript +interface ToolMessage { + id: string; + role: "tool"; + content: string; + toolCallId: string; + error?: string | null; + encryptedValue?: string | null; +} +``` + +### Other Message Types + +- **DeveloperMessage**: `role: "developer"` +- **ReasoningMessage**: `role: "reasoning"` +- **ActivityMessage**: `role: "activity"` (for progress updates) + +--- + +## Tool Schema + +```typescript +interface Tool { + name: string; + description: string; + parameters: object; // JSON Schema 格式 +} +``` + +### Example + +```json +{ + "name": "get_weather", + "description": "Get current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } +} +``` + +### Backend Processing + +Backend 使用 `build_tools_prompt` 函数将 tools 转换为 prompt 格式： + +``` + +- get_weather: Get current weather for a location + - args_schema: {"type":"object","properties":{"location":{"type":"string","description":"City name"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location"]} +- searchDocuments: Search for documents + - args_schema: {"type":"object","properties":{"query":{"type":"string"}},"required":["query"]} +Note: tool arguments must strictly match args_schema. + +``` + +--- + +## Context Schema + +```typescript +interface Context { + description: string; + value: string; +} +``` + +--- + +## Validation Rules + +Backend 实现了以下验证规则： + +| Rule | Error Message | +|------|---------------| +| payload ≤ 256KB | `RunAgentInput payload exceeds size limit` | +| threadId 必须是 UUID | `threadId must be a valid UUID` | +| runId 最大 128 字符 | `runId exceeds length limit` | +| messages ≤ 200 条 | `RunAgentInput.messages exceeds limit` | +| user text ≤ 10,000 字符 | `RunAgentInput user message text exceeds limit` | +| **恰好 1 条 user message** | `RunAgentInput.messages must contain exactly one user message` | +| user message 必须在第一条 | `RunAgentInput.messages[0].role must be user` | +| binary 必须是 image/* | `binary content requires image mimeType` | +| binary 必须有 url | `binary content requires url` | +| binary 不允许使用 data | `binary content data is not allowed` | + +--- + +## Request Example + +### 纯文本请求 + +```json +{ + "threadId": "550e8400-e29b-41d4-a716-446655440000", + "runId": "run-001", + "state": {}, + "messages": [ + { + "id": "msg-001", + "role": "user", + "content": "帮我查一下北京今天的天气" + } + ], + "tools": [], + "context": [], + "forwardedProps": {} +} +``` + +### 多模态请求 (带图片) + +```json +{ + "threadId": "550e8400-e29b-41d4-a716-446655440000", + "runId": "run-002", + "state": {}, + "messages": [ + { + "id": "msg-001", + "role": "user", + "content": [ + { + "type": "text", + "text": "这张图片里的内容是什么?" + }, + { + "type": "binary", + "mimeType": "image/png", + "url": "https://storage.example.com/agent-inputs/user-123/image.png?signature=xxx" + } + ] + } + ], + "tools": [], + "context": [], + "forwardedProps": {} +} +``` + +### 带 Tools 的请求 + +```json +{ + "threadId": "550e8400-e29b-41d4-a716-446655440000", + "runId": "run-003", + "state": {}, + "messages": [ + { + "id": "msg-001", + "role": "user", + "content": "北京天气怎么样?" + } + ], + "tools": [ + { + "name": "get_weather", + "description": "获取指定城市的天气信息", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "城市名称" + } + }, + "required": ["city"] + } + } + ], + "context": [], + "forwardedProps": {} +} +``` + +--- + +## Response + +成功响应返回 `TaskAcceptedResponse`： + +```typescript +interface TaskAcceptedResponse { + taskId: string; + threadId: string; + runId: string; + created: string; // ISO-8601 timestamp +} +``` + +--- + +## History API + +`GET /api/v1/agent/history` 返回对话历史快照。 + +### Request + +| Parameter | Type | Description | +|-----------|------|-------------| +| `threadId` | string (query) | 可选，指定会话 ID，不指定则返回最新会话 | +| `before` | string (query) | 可选，日期格式 `YYYY-MM-DD`，返回该日期之前的快照 | + +### Response + +返回 `HistorySnapshotResponse`： + +```typescript +interface HistorySnapshotResponse { + scope: "history_day"; + threadId: string | null; + day: string | null; // ISO date format "YYYY-MM-DD" + hasMore: boolean; + messages: HistoryMessage[]; +} +``` + +### HistoryMessage + +根据消息 role 不同，返回字段有所差异： + +```typescript +// role = "user" +interface HistoryMessageUser { + id: string; + seq: number; + role: "user"; + content: string; + url: string | null; // 附件临时访问 URL + timestamp: string; // ISO-8601 timestamp +} + +// role = "tool" +interface HistoryMessageTool { + id: string; + seq: number; + role: "tool"; + content: string; + uiSchema: UiSchemaRenderer | null; // 由 tool_agent_output.ui_hints 编译 + timestamp: string; +} + +// role = "assistant" +interface HistoryMessageAssistant { + id: string; + seq: number; + role: "assistant"; + content: string; + uiSchema: UiSchemaRenderer | null; // 由 worker_agent_output.ui_hints 编译 + timestamp: string; +} + +type HistoryMessage = HistoryMessageUser | HistoryMessageTool | HistoryMessageAssistant; +``` + +### UiSchemaRenderer + +编译后的 UI 渲染结构，详见 [UiSchema Protocol](../ui/ui-schema.md)： + +```typescript +interface UiSchemaRenderer { + version: "2.0"; + locale: string; + status: "info" | "success" | "warning" | "error" | "pending"; + theme: "default" | "light" | "dark"; + meta?: { + requestId?: string; + toolId?: string; + traceId?: string; + userId?: string; + }; + root: UiLayoutNode; +} +``` + +### Example + +```json +{ + "scope": "history_day", + "threadId": "550e8400-e29b-41d4-a716-446655440000", + "day": "2026-03-15", + "hasMore": true, + "messages": [ + { + "id": "msg-001", + "seq": 1, + "role": "user", + "content": "帮我创建一个日程", + "url": null, + "timestamp": "2026-03-15T10:00:00Z" + }, + { + "id": "msg-002", + "seq": 2, + "role": "assistant", + "content": "好的，我来帮您创建日程。", + "uiSchema": { + "version": "2.0", + "locale": "zh-CN", + "status": "success", + "theme": "default", + "root": { + "type": "stack", + "appearance": "card", + "children": [ + {"type": "text", "content": "日程已创建", "role": "title"}, + {"type": "badge", "label": "SUCCESS", "status": "success"}, + {"type": "text", "content": "您的会议日程创建成功", "role": "body"} + ] + } + }, + "timestamp": "2026-03-15T10:00:05Z" + } + ] +} +``` + +--- + +## Compatibility Notes + +- `UserMessage.content` 支持 string 或 array 格式，前端优先使用 array 格式以支持多模态 +- binary content 的 url 必须是有效的 signed URL，由 `/api/v1/agent/attachments` 端点生成 +- backend 验证通过后，会将 binary url 转换为内部存储路径 +- tools 为空数组时，prompt 中不会包含工具说明 diff --git a/docs/protocols/agent/sse-events.md b/docs/protocols/agent/sse-events.md new file mode 100644 index 0000000..c6558ab --- /dev/null +++ b/docs/protocols/agent/sse-events.md @@ -0,0 +1,370 @@ +# Agent SSE Events + +本文档记录 Agent Runtime 产生的所有 Server-Sent Events (SSE) 事件，用于前端实时展示。 + +## 事件流转架构 + +``` +pipeline.emit() + ↓ +AgentScopeEventPipeline.emit() + ├─→ store.persist() → 持久化到数据库 + └─→ bus.publish() → 发布到 Redis Stream + ↓ +前端通过 GET /runs/{thread_id}/events 读取 +``` + +## 事件统一格式 + +所有事件在 Redis 中传输时都包含以下字段： + +```typescript +{ + type: string, // 事件类型 + threadId: string, // 会话 ID + runId: string, // 运行 ID + data: object // 事件数据 +} +``` + +--- + +## 1. Orchestrator 生命周期事件 + +### run.started + +Agent 开始运行时触发。 + +```typescript +{ + type: "run.started", + threadId: "xxx", + runId: "yyy", + data: {} +} +``` + +### run.finished + +Agent 成功完成时触发。 + +```typescript +{ + type: "run.finished", + threadId: "xxx", + runId: "yyy", + data: {} +} +``` + +### run.error + +Agent 运行出错时触发。 + +```typescript +{ + type: "run.error", + threadId: "xxx", + runId: "yyy", + data: { + message: "runtime execution failed" + } +} +``` + +--- + +## 2. Step 阶段事件 + +### step.start + +阶段开始时触发。 + +```typescript +{ + type: "step.start", + threadId: "xxx", + runId: "yyy", + data: { + stepName: "router" | "worker" + } +} +``` + +### step.finish + +阶段结束时触发。 + +```typescript +{ + type: "step.finish", + threadId: "xxx", + runId: "yyy", + data: { + stepName: "router" | "worker" + } +} +``` + +--- + +## 3. Worker 运行时事件 + +### 3.1 文本消息事件 + +#### text.start + +文本消息开始时触发。 + +```typescript +{ + type: "text.start", + threadId: "xxx", + runId: "yyy", + data: { + messageId: "msg-xxx", + role: "assistant", + stage: "worker" + } +} +``` + +#### text.delta + +文本内容增量更新时触发。 + +```typescript +{ + type: "text.delta", + threadId: "xxx", + runId: "yyy", + data: { + messageId: "msg-xxx", + delta: "这是新增的文本内容", + stage: "worker" + } +} +``` + +#### text.end + +文本消息结束时触发，包含完整输出和使用统计。 + +```typescript +{ + type: "text.end", + threadId: "xxx", + runId: "yyy", + data: { + messageId: "msg-xxx", + role: "assistant", + stage: "worker", + workerAgentOutput: { + status: "success" | "partial_success" | "failed", + answer: "主回复文本", + key_points: ["要点1", "要点2"], + result_type: "execution_report" | "clarification" | "error_report" | "unknown", + suggested_actions: ["建议操作1"], + error: null | { code: string, message: string }, + ui_hints: { ... } | null // 仅在 ui_mode 非空时存在 + }, + model: "gpt-4o", + inputTokens: 1000, + outputTokens: 500, + cost: 0.025, + latencyMs: 1500 + } +} +``` + +**workerAgentOutput 详细结构：** + +```typescript +// WorkerAgentOutputLite +{ + status: "success" | "partial_success" | "failed", + answer: string, + key_points: string[], + result_type: "execution_report" | "clarification" | "error_report" | "unknown", + suggested_actions: string[], + error: { code: string, message: string } | null +} + +// WorkerAgentOutputRich (当 ui.ui_mode 非空时) +{ + // ... WorkerAgentOutputLite 字段 + ui_hints: { + ui_mode: string, + ui_state: object, + actions: UiHintAction[] + } +} +``` + +--- + +### 3.2 工具调用事件 + +#### tool.start + +工具调用开始时触发。 + +```typescript +{ + type: "tool.start", + threadId: "xxx", + runId: "yyy", + data: { + messageId: "msg-xxx", + toolCallId: "call-abc", + toolName: "calendar_read", + stage: "worker" + } +} +``` + +#### tool.args + +工具调用参数时触发。 + +```typescript +{ + type: "tool.args", + threadId: "xxx", + runId: "yyy", + data: { + messageId: "msg-xxx", + toolCallId: "call-abc", + toolName: "calendar_read", + args: { start_date: "2024-01-01", end_date: "2024-01-07" }, + stage: "worker" + } +} +``` + +#### tool.end + +工具调用结束时触发。 + +```typescript +{ + type: "tool.end", + threadId: "xxx", + runId: "yyy", + data: { + messageId: "msg-xxx", + toolCallId: "call-abc", + toolName: "calendar_read", + stage: "worker" + } +} +``` + +#### tool.result + +工具执行结果时触发。 + +```typescript +{ + type: "tool.result", + threadId: "xxx", + runId: "yyy", + data: { + messageId: "msg-xxx", + toolCallId: "call-abc", + toolName: "calendar_read", + stage: "worker", + toolAgentOutput: { + tool_name: "calendar_read", + tool_call_id: "call-abc", + tool_call_args: { start_date: "2024-01-01", end_date: "2024-01-07" }, + status: "success" | "failed", + result_summary: "找到3个事件...", + ui_hints: null, + attachments: null + } + } +} +``` + +**toolAgentOutput 详细结构：** + +```typescript +{ + tool_name: string, + tool_call_id: string, + tool_call_args: object | null, + status: "success" | "failed", + result_summary: string, + ui_hints: object | null, + attachments: Array<{ + bucket: string, + path: string, + mime_type: string, + url: string + }> | null +} +``` + +--- + +## 使用统计字段 + +`text.end` 事件中包含使用统计： + +| 字段 | 类型 | 说明 | +|------|------|------| +| `model` | string | 使用的模型名称 | +| `inputTokens` | number | 输入 token 数量 | +| `outputTokens` | number | 输出 token 数量 | +| `cost` | number | 花费（美元） | +| `latencyMs` | number | 延迟（毫秒） | + +--- + +## 前端接收示例 + +```javascript +const eventSource = new EventSource(`/runs/${threadId}/events`); + +eventSource.addEventListener('run.started', (e) => { + console.log('Agent started:', JSON.parse(e.data)); +}); + +eventSource.addEventListener('step.start', (e) => { + console.log('Step started:', JSON.parse(e.data)); +}); + +eventSource.addEventListener('text.delta', (e) => { + const data = JSON.parse(e.data); + console.log('Text delta:', data.data.delta); +}); + +eventSource.addEventListener('tool.start', (e) => { + const data = JSON.parse(e.data); + console.log('Tool called:', data.data.toolName); +}); + +eventSource.addEventListener('tool.result', (e) => { + const data = JSON.parse(e.data); + console.log('Tool result:', data.data.toolAgentOutput); +}); + +eventSource.addEventListener('text.end', (e) => { + const data = JSON.parse(e.data); + console.log('Worker output:', data.data.workerAgentOutput); + console.log('Usage:', { + inputTokens: data.data.inputTokens, + outputTokens: data.data.outputTokens, + cost: data.data.cost + }); +}); + +eventSource.addEventListener('run.finished', (e) => { + console.log('Agent finished:', JSON.parse(e.data)); +}); + +eventSource.addEventListener('run.error', (e) => { + console.log('Agent error:', JSON.parse(e.data)); +}); +``` diff --git a/docs/protocols/inbox-messages.md b/docs/protocols/inbox-messages.md deleted file mode 100644 index 4a983ee..0000000 --- a/docs/protocols/inbox-messages.md +++ /dev/null @@ -1,130 +0,0 @@ -# Inbox Messages Protocol - -> **NOTE**: This document is the single source of truth. All implementations must follow this specification. - -## Overview - -Inbox messages are notifications sent to users for various events like friend requests and calendar invites. - -## Version - -- **Current**: `1.0` -- **Status**: Active - ---- - -## Message Types - -```typescript -type InboxMessageType = 'friend_request' | 'calendar' | 'system' | 'group'; -``` - -```typescript -type InboxMessageStatus = 'pending' | 'accepted' | 'rejected' | 'dismissed'; -``` - ---- - -## Content Schema - -### Calendar Invite Content - -```typescript -interface CalendarInviteContent { - type: 'invite'; - permission: number; // 1=view, 4=edit, 8=invite - action: 'pending'; -} -``` - -### Calendar Update Content - -```typescript -interface CalendarUpdateContent { - type: 'update'; - title: string; - action: 'updated'; -} -``` - -### Calendar Delete Content - -```typescript -interface CalendarDeleteContent { - type: 'delete'; - title: string; - action: 'deleted'; -} -``` - -### Friendship Request Content - -```typescript -interface FriendshipContent { - type: 'request'; - message?: string; // Optional friend request message -} -``` - ---- - -## Union Type - -```typescript -type InboxMessageContent = - | CalendarInviteContent - | CalendarUpdateContent - | CalendarDeleteContent - | FriendshipContent; -``` - ---- - -## Database Field - -| Field | Type | Description | -|-------|------|-------------| -| content | jsonb | Structured content based on message_type | - ---- - -## JSON Examples - -### Friend Request - -```json -{ - "type": "request", - "message": "Hi, let's be friends!" -} -``` - -### Calendar Invite - -```json -{ - "type": "invite", - "permission": 4, - "action": "pending" -} -``` - -### Calendar Update - -```json -{ - "type": "update", - "title": "Team Meeting", - "action": "updated" -} -``` - -### Calendar Delete - -```json -{ - "type": "delete", - "title": "Team Meeting", - "action": "deleted" -} -``` diff --git a/docs/protocols/invite-codes.md b/docs/protocols/invite-codes.md deleted file mode 100644 index 399d060..0000000 --- a/docs/protocols/invite-codes.md +++ /dev/null @@ -1,53 +0,0 @@ -# Invite Codes Protocol - -> **NOTE**: This document is the single source of truth. All implementations must follow this specification. - -## Overview - -Invite codes for referral system with reward configuration. - -## Version - -- **Current**: `1.0` -- **Status**: Active - ---- - -## Reward Configuration - -```typescript -interface InviteCodeRewardConfig { - // Reserved for future use - // Currently unused, allowing custom extensions - [key: string]: any; -} -``` - ---- - -## Database Field - -| Field | Type | Description | -|-------|------|-------------| -| reward_config | jsonb | Reward configuration for invite codes | - ---- - -## JSON Examples - -### Empty Config - -```json -{} -``` - -### Future Usage Example - -```json -{ - "reward_type": "credits", - "reward_amount": 100, - "currency": "USD", - "max_rewards_per_user": 5 -} -``` diff --git a/docs/protocols/memories.md b/docs/protocols/memories.md deleted file mode 100644 index c604743..0000000 --- a/docs/protocols/memories.md +++ /dev/null @@ -1,66 +0,0 @@ -# Memories Protocol - -> **NOTE**: This document is the single source of truth. All implementations must follow this specification. - -## Overview - -User memories stored in the system with flexible content structure. - -## Version - -- **Current**: `1.0` -- **Status**: Active - ---- - -## Memory Content - -```typescript -interface MemoryContent { - // Reserved for future use - // Currently unused, allowing custom extensions - [key: string]: any; -} -``` - ---- - -## Database Field - -| Field | Type | Description | -|-------|------|-------------| -| content | jsonb | Memory content with flexible structure | - ---- - -## JSON Examples - -### Empty Content - -```json -{} -``` - -### Future Usage Example - -```json -{ - "text": "Remember that user prefers morning meetings", - "category": "preference", - "importance": "high", - "tags": ["meetings", "morning", "preference"] -} -``` - -### Agent Memory Example - -```json -{ - "summary": "User discussed vacation plans for March", - "entities": { - "dates": ["2026-03-15", "2026-03-20"], - "locations": ["Tokyo", "Osaka"] - }, - "sentiment": "positive" -} -``` diff --git a/docs/protocols/profiles.md b/docs/protocols/profiles.md deleted file mode 100644 index 45a9a41..0000000 --- a/docs/protocols/profiles.md +++ /dev/null @@ -1,88 +0,0 @@ -# Profiles Protocol - -> **NOTE**: This document is the single source of truth. All implementations must follow this specification. - -## Overview - -User profile settings with privacy and notification preferences. - -## Version - -- **Current**: `1.0` -- **Status**: Active - ---- - -## Preference Settings - -### Privacy - -```typescript -interface PrivacySettings { - show_email?: boolean; - show_online_status?: boolean; -} -``` - -### Notification - -```typescript -interface NotificationSettings { - friend_request?: boolean; - calendar_invite?: boolean; - calendar_update?: boolean; - calendar_delete?: boolean; - system_message?: boolean; -} -``` - ---- - -## Profile Settings V1 - -```typescript -interface ProfileSettingsV1 { - privacy?: PrivacySettings; - notification?: NotificationSettings; -} -``` - ---- - -## Database Field - -| Field | Type | Description | -|-------|------|-------------| -| settings | jsonb | User preferences including privacy and notification settings | - ---- - -## JSON Examples - -### Default Settings - -```json -{ - "privacy": { - "show_email": false, - "show_online_status": true - }, - "notification": { - "friend_request": true, - "calendar_invite": true, - "calendar_update": true, - "calendar_delete": true, - "system_message": true - } -} -``` - -### Minimal Settings - -```json -{ - "notification": { - "friend_request": false - } -} -``` diff --git a/docs/protocols/routes.md b/docs/protocols/routes.md deleted file mode 100644 index 0eb82d1..0000000 --- a/docs/protocols/routes.md +++ /dev/null @@ -1,13 +0,0 @@ -# Auth Routes Protocol Notes - -## POST `/api/v1/auth/verifications` - -- `invite_code` is optional. -- Recommended format is fixed `4` chars and pattern `^[ABCDEFGHJKMNPQRSTUVWXYZ23456789]{4}$`. -- Backend normalizes invite codes to uppercase and validates in service logic. -- Invalid invite code values are ignored (treated as empty), and signup verification email flow still continues. - -## Verification Token Input Convention - -- Verification token for signup/recovery uses fixed `6` digits. -- Client UI should use fixed-length segmented input to reduce mistyped values. diff --git a/docs/protocols/routes/agent-runs-events-history.md b/docs/protocols/routes/agent-runs-events-history.md deleted file mode 100644 index cea400a..0000000 --- a/docs/protocols/routes/agent-runs-events-history.md +++ /dev/null @@ -1,239 +0,0 @@ -# Agent Runs Events and History Route Protocol - -> **NOTE**: This document is the single source of truth for agent runs event streaming and history snapshot routes. - -## Overview - -Defines the transport format for: - -- `POST /api/v1/agent/runs` -- `GET /api/v1/agent/runs/{thread_id}/events` -- `GET /api/v1/agent/history` -- `GET /api/v1/agent/attachments/signed-url` - -## Version - -- **Current**: `1.0` -- **Status**: Draft (pending full backend/frontend alignment) - ---- - -## Route Semantics - -### `GET /api/v1/agent/history` - -- Unified history endpoint. -- Query params: - - `threadId` (optional): target thread id. - - `before` (optional, `YYYY-MM-DD`): paginate by day. -- Behavior: - - With `threadId`: returns that thread's day snapshot. - - Without `threadId`: returns latest available thread snapshot for current user. - -### `GET /api/v1/agent/attachments/signed-url` - -- Generate temporary signed URL for attachment rendering. -- Query params: - - `bucket` (required) - - `path` (required) -- Scope rule: - - `bucket` must match current storage bucket. - - `path` must be within current user prefix `agent-inputs/{user_id}/`. - ---- - -## SSE Envelope (`/events`) - -`GET /api/v1/agent/runs/{thread_id}/events` uses `text/event-stream`. - -Each SSE frame format: - -```text -id: -event: -data: - -``` - ---- - -## Event Type Set - -- `RUN_STARTED` -- `STEP_STARTED` -- `STEP_FINISHED` -- `TEXT_MESSAGE_START` -- `TEXT_MESSAGE_CONTENT` -- `TEXT_MESSAGE_END` -- `TOOL_CALL_RESULT` -- `RUN_FINISHED` -- `RUN_ERROR` - ---- - -## Common Event Fields - -```typescript -interface EventBase { - type: string; - threadId: string; - runId?: string; -} -``` - ---- - -## Event Payload Schemas - -### Run Lifecycle - -```typescript -interface RunStartedEvent extends EventBase { - type: "RUN_STARTED"; - runId: string; -} - -interface RunFinishedEvent extends EventBase { - type: "RUN_FINISHED"; - runId: string; -} - -interface RunErrorEvent extends EventBase { - type: "RUN_ERROR"; - runId: string; - message: string; -} -``` - -### Step Lifecycle - -```typescript -interface StepStartedEvent extends EventBase { - type: "STEP_STARTED"; - runId: string; - stepName: string; -} - -interface StepFinishedEvent extends EventBase { - type: "STEP_FINISHED"; - runId: string; - stepName: string; -} -``` - -### Text Streaming - -```typescript -interface TextMessageStartEvent extends EventBase { - type: "TEXT_MESSAGE_START"; - runId: string; - messageId: string; - role: "assistant" | "system" | "user" | "tool"; - stage?: string; -} - -interface TextMessageContentEvent extends EventBase { - type: "TEXT_MESSAGE_CONTENT"; - runId: string; - messageId: string; - delta: string; // incremental text chunk -} - -interface TextMessageEndEvent extends EventBase { - type: "TEXT_MESSAGE_END"; - runId: string; - messageId: string; - workerAgentOutput: WorkerAgentOutput; - // stage/model are intentionally excluded from this event -} -``` - -### Tool Result - -```typescript -interface ToolCallResultEvent extends EventBase { - type: "TOOL_CALL_RESULT"; - messageId: string; - toolCallId: string; - toolAgentOutput: ToolAgentOutput; // required -} -``` - -### Worker/Tool Payloads - -```typescript -interface WorkerAgentOutput { - status: "success" | "partial_success" | "failed"; - answer: string; - key_points?: string[]; - result_type?: string; - suggested_actions?: string[]; - error?: { - code: string; - message: string; - retryable?: boolean; - details?: Record; - }; - ui_hints?: Record; -} - -interface ToolAgentOutput { - tool_name: string; - tool_call_id: string; - tool_call_args?: Record; - status: "success" | "partial" | "failure"; - result_summary: string; - ui_hints?: Record; - error?: { - code: string; - message: string; - retryable?: boolean; - details?: Record; - }; -} -``` - ---- - -## History Response Schema - -`GET /api/v1/agent/history` returns `STATE_SNAPSHOT` payload. - -```typescript -interface AgentHistoryResponse { - type: "STATE_SNAPSHOT"; - threadId?: string; - snapshot: { - scope: "history_day"; - threadId: string | null; - day: string | null; // YYYY-MM-DD - hasMore: boolean; - messages: SnapshotMessage[]; - }; -} - -interface SnapshotMessage { - id: string; - seq: number; - role: "user" | "assistant" | "system" | "tool"; - content: string; - metadata?: Record; - timestamp: string; // ISO-8601 -} - -interface AttachmentSignedUrlResponse { - bucket: string; - path: string; - url: string; -} -``` - ---- - -## Compatibility Notes - -- For `TOOL_CALL_RESULT`, clients should treat `toolAgentOutput` as canonical payload. -- `TEXT_MESSAGE_CONTENT.delta` is defined as incremental text chunk. Implementations should emit multiple chunks for real streaming UX. -- `TEXT_MESSAGE_END` must not include `stage` or `model` in this protocol version. -- History snapshot `messages[]` strictly follows `backend/src/schemas/messages/chat_message.py` `AgentChatMessage` schema. -- Attachment URL rendering is decoupled from history; client should call `/api/v1/agent/attachments/signed-url` using metadata fields. diff --git a/docs/protocols/schedule-items.md b/docs/protocols/schedule-items.md deleted file mode 100644 index f614ac0..0000000 --- a/docs/protocols/schedule-items.md +++ /dev/null @@ -1,106 +0,0 @@ -# Schedule Items Protocol - -> **NOTE**: This document is the single source of truth. All implementations must follow this specification. - -## Overview - -Schedule items represent calendar events with metadata, attachments, and sharing capabilities. - -## Version - -- **Current**: `1.0` -- **Status**: Active - ---- - -## Status - -```typescript -type ScheduleItemStatus = 'active' | 'completed' | 'canceled' | 'archived'; -``` - -## Source Type - -```typescript -type ScheduleItemSourceType = 'manual' | 'imported' | 'agent_generated'; -``` - ---- - -## Metadata Schema - -### Attachment Type - -```typescript -type AttachmentType = 'document' | 'reminder'; -``` - -### Attachment - -```typescript -interface ScheduleItemMetadataAttachment { - name: string; - type: AttachmentType; - visible_to: string[]; // UUIDs - url?: string; - note?: string; - content?: string; -} -``` - -### Metadata - -```typescript -interface ScheduleItemMetadata { - color?: string; // "#RRGGBB" format - location?: string; - notes?: string; - attachments?: ScheduleItemMetadataAttachment[]; - reminder_minutes?: number; // 0-10080 (0 to 7 days) - version: 1; -} -``` - ---- - -## Database Field - -| Field | Type | Description | -|-------|------|-------------| -| metadata | jsonb | Structured metadata including color, location, notes, attachments, reminders | - ---- - -## JSON Examples - -### Basic Metadata - -```json -{ - "color": "#3B82F6", - "location": "Conference Room A", - "notes": "Bring presentation slides", - "reminder_minutes": 15, - "version": 1 -} -``` - -### With Attachment - -```json -{ - "color": "#10B981", - "location": "https://meet.example.com/abc123", - "notes": "Weekly sync meeting", - "attachments": [ - { - "name": "agenda.pdf", - "type": "document", - "url": "https://storage.example.com/agenda.pdf", - "visible_to": ["uuid1", "uuid2"] - } - ], - "reminder_minutes": 30, - "version": 1 -} -``` diff --git a/docs/protocols/system-agents.md b/docs/protocols/system-agents.md deleted file mode 100644 index b8a3e5e..0000000 --- a/docs/protocols/system-agents.md +++ /dev/null @@ -1,66 +0,0 @@ -# System Agents Protocol - -> **NOTE**: This document is the single source of truth. All implementations must follow this specification. - -## Overview - -System agent configuration for LLM parameters. - -## Version - -- **Current**: `1.0` -- **Status**: Active - ---- - -## LLM Configuration - -```typescript -interface SystemAgentLLMConfig { - temperature?: number; // 0.0 - 2.0 - max_tokens?: number; // >= 1 - timeout_seconds?: number; // > 0, <= 300, default 30 -} -``` - ---- - -## Database Field - -| Field | Type | Description | -|-------|------|-------------| -| config | jsonb | LLM configuration including temperature, max_tokens, timeout | - ---- - -## JSON Examples - -### Default Configuration - -```json -{ - "temperature": 0.7, - "max_tokens": null, - "timeout_seconds": 30 -} -``` - -### Creative Configuration - -```json -{ - "temperature": 1.2, - "max_tokens": 2000, - "timeout_seconds": 60 -} -``` - -### Precise Configuration - -```json -{ - "temperature": 0.1, - "max_tokens": 500, - "timeout_seconds": 30 -} -``` diff --git a/docs/protocols/ui-schema.md b/docs/protocols/ui-schema.md deleted file mode 100644 index cbff018..0000000 --- a/docs/protocols/ui-schema.md +++ /dev/null @@ -1,466 +0,0 @@ -# UI Schema Protocol - -> **NOTE**: This document is the single source of truth. All implementations must follow this specification. - -## Overview - -A generic UI schema for rendering tool/agent execution results. Designed for AI Agent / Tool ecosystem with extensibility. - -## Version - -- **Current**: `1.0` -- **Status**: Frozen (no new node types) - -## Architecture - -``` -┌─────────────────────────────────────────────────────────────┐ -│ UiSchemaDocument (root) │ -│ - version / schemaType / status / docId │ -│ - meta (protocol-level metadata) │ -│ - nodes (array of UiNode) │ -├─────────────────────────────────────────────────────────────┤ -│ Field Layers: │ -│ 1. Public fields (all renderers must handle) │ -│ id / type / title / description / icon / status / │ -│ timestamp / actions │ -│ 2. meta (protocol-level, not rendered) │ -│ requestId / toolId / traceId / userId │ -│ 3. extensions (tool私有扩展, renderer透传) │ -└─────────────────────────────────────────────────────────────┘ -``` - ---- - -## Data Types - -### SchemaType - -```typescript -type SchemaType = 'tool_result' | 'agent_response' | 'notification'; -``` - -### UiStatus - -```typescript -type UiStatus = 'info' | 'success' | 'warning' | 'error' | 'pending'; -``` - -### IconSource - -```typescript -type IconSource = 'icon' | 'emoji' | 'url'; -``` - -### ActionType - -```typescript -type ActionType = 'navigation' | 'url' | 'event' | 'tool' | 'copy' | 'payload'; -``` - ---- - -## Root Structure - -```typescript -interface UiSchemaDocument { - // Protocol identifier - version: string; // "1.0" - schemaType: SchemaType; // tool_result | agent_response | notification - - // Document metadata - docId?: string; // For local refresh / diff / analytics - timestamp?: string; // ISO 8601 - locale?: string; // "zh-CN" - - // Unified status - status: UiStatus; - - // Render control - renderer?: { - renderer?: string; // Dedicated renderer name - theme?: 'default' | 'dark' | 'light'; - }; - - // Protocol-level metadata (not rendered) - meta?: { - requestId?: string; - toolId?: string; - traceId?: string; - userId?: string; - [key: string]: any; - }; - - // Root nodes - nodes: UiNode[]; -} -``` - ---- - -## Node Types (v1 Whitelist) - -``` -✅ Supported in v1: - - card 卡片 - - list 列表 - - table 表格 - - text 文本/Markdown - - kv 键值对 - - operation 操作结果 - - error 错误提示 - - container 容器 - -❌ Not supported in v1 (reserved for v2): - - chart / metric / image / video / tabs / accordion / form -``` - ---- - -## Common Node Fields - -All nodes share these fields: - -```typescript -interface UiBaseNode { - id?: string; // For local refresh / action targeting - type: string; // Node type identifier -} - -interface UiTitledNode extends UiBaseNode { - title?: string; - description?: string; - icon?: UiIcon; - status?: UiStatus; - timestamp?: string; -} - -interface UiActionableNode extends UiTitledNode { - actions?: UiAction[]; -} - -interface UiExtendableNode extends UiActionableNode { - extensions?: Record; // Tool私有扩展, 通用renderer透传 -} -``` - ---- - -## Node Definitions - -### 1. Card Node - -```typescript -interface UiCardNode extends UiExtendableNode { - type: 'card'; - children?: UiNode[]; // Nested nodes - footer?: UiTextNode; -} -``` - -### 2. List Node - -```typescript -interface UiListNode extends UiExtendableNode { - type: 'list'; - items: ListItem[]; - pagination?: { - page: number; - pageSize: number; - total: number; - hasMore: boolean; - }; - emptyText?: string; -} - -interface ListItem { - id: string; - title: string; - subtitle?: string; - description?: string; - icon?: UiIcon; - badge?: { label: string; variant?: 'default' | 'success' | 'warning' | 'error' | 'info' }; - metadata?: Record; - actions?: UiAction[]; -} -``` - -### 3. Table Node - -```typescript -interface UiTableNode extends UiExtendableNode { - type: 'table'; - columns: TableColumn[]; - rows: TableRow[]; - pagination?: Pagination; -} - -interface TableColumn { - key: string; - label: string; - width?: string; - align?: 'left' | 'center' | 'right'; -} - -interface TableRow { - id: string; - cells: Record; - metadata?: Record; - actions?: UiAction[]; -} -``` - -### 4. Text Node - -```typescript -interface UiTextNode extends UiBaseNode { - type: 'text'; - content: string; - format?: 'plain' | 'markdown'; - icon?: UiIcon; - actions?: UiAction[]; -} -``` - -### 5. Key-Value Node - -```typescript -interface UiKvNode extends UiExtendableNode { - type: 'kv'; - pairs: KeyValuePair[]; - layout?: 'vertical' | 'horizontal' | 'grid'; -} - -interface KeyValuePair { - key: string; - label?: string; - value: string | number | boolean; - copyable?: boolean; -} -``` - -### 6. Operation Node - -```typescript -interface UiOperationNode extends UiExtendableNode { - type: 'operation'; - operation: 'create' | 'update' | 'delete' | 'execute'; - result: 'success' | 'failure' | 'partial'; - message?: string; - affectedCount?: number; - details?: UiNode; - rollback?: UiAction; -} -``` - -### 7. Error Node - -```typescript -interface UiErrorNode extends UiBaseNode { - type: 'error'; - title?: string; - icon?: UiIcon; - errorCode: string; - message: string; - details?: string; - stack?: string; - retryable: boolean; - suggestions?: string[]; - retry?: UiAction; - support?: UiAction; - actions?: UiAction[]; -} -``` - -### 8. Container Node - -```typescript -interface UiContainerNode extends UiBaseNode { - type: 'container'; - direction: 'vertical' | 'horizontal'; - gap?: number; - children: UiNode[]; - actions?: UiAction[]; -} -``` - ---- - -## Action Structure - -```typescript -interface UiAction { - id: string; - label: string; - icon?: UiIcon; - style?: 'primary' | 'secondary' | 'ghost' | 'danger'; - disabled?: boolean; - action: ActionSpec; - confirm?: { - title?: string; - message?: string; - confirmLabel?: string; - cancelLabel?: string; - }; -} - -type ActionSpec = - | { type: 'navigation'; path: string; params?: Record } - | { type: 'url'; url: string; target?: '_self' | '_blank' } - | { type: 'event'; event: string; payload?: Record } - | { type: 'tool'; toolId: string; params?: Record } - | { type: 'copy'; content: string; successMessage?: string } - | { type: 'payload'; payload: Record; submitTo?: string }; -``` - ---- - -## Icon Structure - -```typescript -interface UiIcon { - source: IconSource; // 'icon' | 'emoji' | 'url' - value: string; // icon name / emoji / URL - color?: string; // "#FF0000" - size?: number; // pixels -} -``` - ---- - -## Usage Rules - -### operation vs error - -| Scenario | Node Type | -|----------|-----------| -| Tool execution failed, system exception | `UiErrorNode` | -| Business operation result (CRUD) | `UiOperationNode` | -| Network error / permission denied | `UiErrorNode` | - -### extensions Usage Constraints - -1. ❌ NO: Any rendering-related style / text / layout -2. ❌ NO: Any fields other renderers need to read -3. ✅ YES: Business identifiers (eventId, orderId) -4. ✅ YES: Dedicated renderer private config -5. ✅ YES: Analytics / logging context data -6. ✅ YES: Data that generic renderer doesn't care about - ---- - -## JSON Examples - -### Example 1: Success Card - -```json -{ - "version": "1.0", - "schemaType": "tool_result", - "docId": "doc_evt_001", - "timestamp": "2026-03-12T10:30:00Z", - "status": "success", - "renderer": { "renderer": "calendar" }, - "meta": { "requestId": "req_abc", "toolId": "calendar.create_event" }, - "nodes": [ - { - "id": "node_card_1", - "type": "card", - "title": "日程已创建", - "description": "会议日程创建成功", - "icon": { "source": "icon", "value": "event_available" }, - "extensions": { "eventId": "evt_abc123", "color": "#3B82F6" }, - "children": [ - { - "type": "kv", - "pairs": [ - { "key": "title", "label": "主题", "value": "Q1 规划会议" }, - { "key": "time", "label": "时间", "value": "2026-03-15 14:00" } - ] - } - ], - "actions": [ - { - "id": "action_view", - "label": "查看详情", - "style": "primary", - "action": { "type": "navigation", "path": "/calendar/evt_abc123" } - } - ] - } - ] -} -``` - -### Example 2: List Result - -```json -{ - "version": "1.0", - "schemaType": "tool_result", - "status": "success", - "meta": { "toolId": "search.documents" }, - "nodes": [ - { "type": "text", "content": "找到 **3** 个相关文档" }, - { - "id": "node_list_1", - "type": "list", - "items": [ - { - "id": "item_1", - "title": "API 设计规范", - "subtitle": "v2.1", - "icon": { "source": "emoji", "value": "📄" }, - "actions": [ - { "id": "a1", "label": "查看", "action": { "type": "url", "url": "/docs/api" } } - ] - } - ], - "pagination": { "page": 1, "pageSize": 10, "total": 3, "hasMore": false } - } - ] -} -``` - -### Example 3: Error Result - -```json -{ - "version": "1.0", - "schemaType": "tool_result", - "status": "error", - "meta": { "toolId": "user.delete" }, - "nodes": [ - { - "id": "node_error_1", - "type": "error", - "title": "删除用户失败", - "icon": { "source": "icon", "value": "error_outline" }, - "errorCode": "PERMISSION_DENIED", - "message": "您没有权限执行此操作", - "details": "需要管理员权限", - "retryable": true, - "suggestions": ["联系管理员", "联系技术支持"], - "retry": { - "id": "action_retry", - "label": "重试", - "style": "primary", - "action": { "type": "tool", "toolId": "user.delete", "params": { "userId": "u1" } } - } - } - ] -} -``` - ---- - -## Evolution (v2) - -Reserved for future: - -- Nested containers: `tabs`, `accordion`, `carousel` -- Data visualization: `chart`, `metric`, `progress` -- Rich media: `image`, `video`, `audio`, `file` -- Internationalization: `i18nKey` field -- Version migration: `deprecated` flag -- Offline support: `offline` flag diff --git a/docs/protocols/ui/data-flow.md b/docs/protocols/ui/data-flow.md new file mode 100644 index 0000000..a139aaa --- /dev/null +++ b/docs/protocols/ui/data-flow.md @@ -0,0 +1,432 @@ +# 前后端数据流通指南 + +本文档描述前端如何与后端 Agent 系统交互，以及如何渲染后端返回的 UI 数据。 + +--- + +## 1. 整体交互流程 + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ 前端 │ +├─────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────┐ POST /runs ┌──────────────┐ │ +│ │ 构造请求 │ ──────────────────▶│ 后端 │ │ +│ │ (多模态数据) │ │ (任务入队) │ │ +│ └──────────────┘◀────────────────── └──────────────┘ 202 Accepted │ +│ │ taskId │ +│ │ │ +│ │ GET /runs/{threadId}/events │ +│ │──────────────────────────────────────────────────────────▶ │ +│ │ SSE 事件流 ◀───────────────── │ +│ │ │ +│ ┌─────┴──────────────────────────────────────────────────────────┐ │ +│ │ 渲染事件 │ │ +│ │ - text.delta: 追加文本 │ │ +│ │ - tool.result: 渲染 toolAgentOutput.ui_hints → UiSchema │ │ +│ │ - text.end: 渲染 workerAgentOutput.ui_hints → UiSchema │ │ +│ └────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────────┐ +│ 后端 │ +├─────────────────────────────────────────────────────────────────────────┤ +│ │ +│ POST /runs │ +│ → 验证请求 │ +│ → 任务入队列 (Taskiq) │ +│ │ +│ Worker 执行 │ +│ → emit 事件到 Redis Stream │ +│ → UiHints → ui_compiler.compile() → UiSchemaRenderer │ +│ │ +│ 持久化 │ +│ → AgentChatMessage (含 uiSchema) │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 2. 多模态数据处理 + +### 2.1 前端上传附件流程 + +```javascript +// 方式一：先上传附件，获取签名 URL +const formData = new FormData(); +formData.append('threadId', threadId); +formData.append('file', imageFile); + +const uploadResponse = await fetch('/api/v1/agent/attachments', { + method: 'POST', + body: formData, +}); +const { attachment } = await uploadResponse.json(); +// attachment.url 是临时访问 URL + +// 方式二：直接使用已有的签名 URL（如果有） +const imageUrl = 'https://storage.example.com/...'; +``` + +### 2.2 构造 RunAgentInput + +```javascript +const runInput = { + threadId: threadId, + runId: `run-${Date.now()}`, + state: {}, + messages: [ + { + id: `msg-${Date.now()}`, + role: 'user', + content: [ + { type: 'text', text: '这张图片里有什么？' }, + { + type: 'binary', + mimeType: 'image/png', + url: attachment.url // 签名 URL + } + ] + } + ], + tools: [], // 或传入工具定义 + context: [], + forwardedProps: {} +}; +``` + +### 2.3 发送请求 + +```javascript +const response = await fetch('/api/v1/agent/runs', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(runInput) +}); +// 返回 202 Accepted +const { taskId, threadId, runId, created } = await response.json(); +``` + +--- + +## 3. 事件流渲染 + +### 3.1 订阅事件 + +```javascript +class AgentEventHandler { + constructor(threadId) { + this.eventSource = new EventSource(`/api/v1/agent/runs/${threadId}/events`); + this.setupListeners(); + } + + setupListeners() { + this.eventSource.addEventListener('run.started', this.onRunStarted); + this.eventSource.addEventListener('step.start', this.onStepStart); + this.eventSource.addEventListener('text.delta', this.onTextDelta); + this.eventSource.addEventListener('text.message.start', this.onTextStart); + this.eventSource.addEventListener('tool.call.start', this.onToolStart); + this.eventSource.addEventListener('tool.call.args', this.onToolArgs); + this.eventSource.addEventListener('tool.call.result', this.onToolResult); + this.eventSource.addEventListener('text.message.end', this.onTextEnd); + this.eventSource.addEventListener('run.finished', this.onRunFinished); + this.eventSource.addEventListener('run.error', this.onRunError); + } +} +``` + +### 3.2 事件处理与渲染 + +#### text.delta - 文本增量 + +```javascript +onTextDelta(event) { + const data = JSON.parse(event.data); + const { delta, messageId } = data.data; + // 追加到对应消息的文本内容 + appendTextToMessage(messageId, delta); +} +``` + +#### tool.result - 工具结果 + +```javascript +onToolResult(event) { + const data = JSON.parse(event.data); + const { toolAgentOutput, toolCallId } = data.data; + + if (toolAgentOutput?.ui_hints) { + // 后端返回的是 UiHints，需要编译为 UiSchemaRenderer + const uiSchema = compileUiHints(toolAgentOutput.ui_hints); + renderUiSchema(uiSchema, toolCallId); + } else { + // 纯文本结果 + renderText(toolAgentOutput.result_summary, toolCallId); + } +} +``` + +#### text.end - Worker 完成 + +```javascript +onTextEnd(event) { + const data = JSON.parse(event.data); + const { workerAgentOutput, inputTokens, outputTokens, cost } = data.data; + + if (workerAgentOutput?.ui_hints) { + // 渲染 UiHints → UiSchemaRenderer + const uiSchema = compileUiHints(workerAgentOutput.ui_hints); + renderUiSchema(uiSchema, messageId); + } else { + // 纯文本回复 + renderText(workerAgentOutput?.answer, messageId); + } + + // 显示使用统计 + showUsageStats({ inputTokens, outputTokens, cost }); +} +``` + +--- + +## 4. History 对话历史渲染 + +### 4.1 获取历史 + +```javascript +const response = await fetch( + `/api/v1/agent/history?threadId=${threadId}&before=${date}` +); +const { messages } = await response.json(); +``` + +### 4.2 渲染历史消息 + +```javascript +messages.forEach(msg => { + switch (msg.role) { + case 'user': + renderUserMessage(msg); + break; + case 'assistant': + if (msg.uiSchema) { + // 直接渲染 UiSchemaRenderer（后端已编译好） + renderUiSchema(msg.uiSchema); + } else { + // 纯文本 + renderText(msg.content); + } + break; + case 'tool': + // tool 结果也可能有 uiSchema + if (msg.uiSchema) { + renderUiSchema(msg.uiSchema); + } else { + renderText(msg.content); + } + break; + } +}); +``` + +--- + +## 5. UiSchemaRenderer 渲染 + +### 5.1 数据结构 + +后端 `ui_compiler.compile()` 输出的结构： + +```typescript +interface UiSchemaRenderer { + version: "2.0"; + locale: string; + status: "info" | "success" | "warning" | "error" | "pending"; + theme: "default" | "light" | "dark"; + meta?: { + requestId?: string; + toolId?: string; + traceId?: string; + userId?: string; + }; + root: UiLayoutNode; +} + +// 布局节点 +interface UiStackNode { + type: "stack"; + direction: "vertical" | "horizontal"; + gap?: number; + appearance: "plain" | "card" | "section"; + status?: UiStatus; + align?: LayoutAlign; + justify?: LayoutJustify; + wrap?: boolean; + children: UiNode[]; +} + +interface UiGridNode { + type: "grid"; + columns: number; + gap?: number; + appearance: LayoutAppearance; + status?: UiStatus; + children: UiNode[]; +} + +// 基础节点 +type UiNode = + | UiTextNode // 文本 + | UiIconNode // 图标 + | UiBadgeNode // 标签 + | UiButtonNode // 按钮 + | UiKvNode // 键值对 + | UiDividerNode; // 分割线 + +// 文本节点 +interface UiTextNode { + type: "text"; + content: string; + format: "plain" | "markdown"; + role: "title" | "subtitle" | "body" | "caption" | "code"; + status?: UiStatus; + maxLines?: number; + visible?: boolean; +} + +// 按钮节点 +interface UiButtonNode { + type: "button"; + label: string; + style: "primary" | "secondary" | "ghost" | "danger"; + disabled?: boolean; + icon?: UiIconSpec; + action: UiActionPayload; +} + +// 键值对节点 +interface UiKvNode { + type: "kv"; + items: UiKvItem[]; + columns?: number; +} + +interface UiKvItem { + key: string; + label?: string; + value: any; + copyable?: boolean; +} + +// Action payloads +type UiActionPayload = + | { type: "navigation"; path: string; params?: Record } + | { type: "url"; url: string; target?: "_self" | "_blank" } + | { type: "event"; event: string; payload?: Record } + | { type: "tool"; toolId: string; params?: Record } + | { type: "copy"; content: string; successMessage?: string } + | { type: "payload"; payload: Record; submitTo?: string }; +``` + +### 5.2 统一渲染器实现 + +```dart +class UiSchemaRendererWidget extends StatelessWidget { + final UiSchemaRenderer schema; + + @override + Widget build(BuildContext context) { + return renderNode(schema.root); + } + + Widget renderNode(UiNode node) { + switch (node.type) { + case 'text': + return renderTextNode(node as UiTextNode); + case 'icon': + return renderIconNode(node as UiIconNode); + case 'badge': + return renderBadgeNode(node as UiBadgeNode); + case 'button': + return renderButtonNode(node as UiButtonNode); + case 'kv': + return renderKvNode(node as UiKvNode); + case 'divider': + return renderDividerNode(node as UiDividerNode); + case 'stack': + return renderStackNode(node as UiStackNode); + case 'grid': + return renderGridNode(node as UiGridNode); + default: + return SizedBox.shrink(); + } + } +} +``` + +### 5.3 事件渲染 vs History 渲染一致性 + +**关键点**：两者使用同一个渲染器 + +| 数据来源 | 数据格式 | 处理方式 | +|---------|---------|---------| +| events text.end | `workerAgentOutput.ui_hints` | compile → UiSchemaRenderer → 渲染 | +| events tool.result | `toolAgentOutput.ui_hints` | compile → UiSchemaRenderer → 渲染 | +| history assistant | `msg.uiSchema` | 直接渲染 | +| history tool | `msg.uiSchema` | 直接渲染 | + +**编译函数**（前端需要实现）： + +```javascript +// 将后端 UiHints 编译为 UiSchemaRenderer +function compileUiHints(hints) { + // 这里可以调用后端的 compile 接口 + // 或者在前端实现相同的编译逻辑 + return fetch('/api/v1/agent/compile-ui-hints', { + method: 'POST', + body: JSON.stringify(hints) + }).then(r => r.json()); +} +``` + +--- + +## 6. UiHints 介绍（可选） + +如果前端需要自己处理 UiHints，以下是结构： + +### UiHintsPayload + +```typescript +interface UiHintsPayload { + version: "2.1"; + intent: "message" | "data" | "list" | "status" | "form" | "mixed"; + status: "info" | "success" | "warning" | "error" | "pending"; + + title?: string; + description?: string; + body?: string; + bodyFormat?: "plain" | "markdown"; + + items?: UiHintKvItem[]; + listItems?: UiHintListItem[]; + sections?: UiHintSection[]; + actions?: UiHintAction[]; + icon?: UiHintIcon; + meta?: Record; +} +``` + +**注意**：History API 返回的消息已经包含了编译好的 `uiSchema`，前端不需要自己编译。 + +--- + +## 7. 总结 + +1. **多模态**：前端上传附件 → 获取签名 URL → 构造 RunAgentInput → POST /runs +2. **实时事件**：订阅 /runs/{threadId}/events → 处理 text.delta/tool.result/text.end → 渲染 UiSchema +3. **历史消息**：GET /history → 遍历消息 → 有 uiSchema 则渲染，否则渲染纯文本 +4. **一致性**：events 和 history 使用同一个 UiSchemaRenderer 渲染组件 diff --git a/docs/protocols/ui/ui-schema.md b/docs/protocols/ui/ui-schema.md new file mode 100644 index 0000000..61357a3 --- /dev/null +++ b/docs/protocols/ui/ui-schema.md @@ -0,0 +1,747 @@ +# UI Schema Protocol + +> **NOTE**: This document is the single source of truth. All implementations must follow this specification. + +## Overview + +A generic UI schema for rendering tool/agent execution results. Designed for AI Agent / Tool ecosystem with extensibility. + +**Design Philosophy**: Keep only "primitive components + layout containers". Frontend only needs to recursively render the root layout tree. + +## Version + +- **Current**: `2.0` +- **Status**: Active + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ UiSchemaRenderer (root) │ +│ - version / locale / status / theme │ +│ - meta (protocol-level metadata) │ +│ - root (UiLayoutNode - stack or grid) │ +├─────────────────────────────────────────────────────────────┤ +│ Rendering Flow: │ +│ 1. Backend returns UiSchemaRenderer with root layout │ +│ 2. Frontend recursively renders root layout tree │ +│ 3. Layout nodes (stack/grid) contain children │ +│ 4. Primitive nodes (text/icon/button/etc) are leaves │ +└─────────────────────────────────────────────────────────────┘ +``` + +--- + +## Data Types + +### UiStatus + +```typescript +type UiStatus = 'info' | 'success' | 'warning' | 'error' | 'pending'; +``` + +### IconSource + +```typescript +type IconSource = 'icon' | 'emoji' | 'url'; +``` + +### TextFormat + +```typescript +type TextFormat = 'plain' | 'markdown'; +``` + +### TextRole + +```typescript +type TextRole = 'title' | 'subtitle' | 'body' | 'caption' | 'code'; +``` + +### ButtonStyle + +```typescript +type ButtonStyle = 'primary' | 'secondary' | 'ghost' | 'danger'; +``` + +### LayoutDirection + +```typescript +type LayoutDirection = 'vertical' | 'horizontal'; +``` + +### LayoutAppearance + +```typescript +type LayoutAppearance = 'plain' | 'card' | 'section'; +``` + +### LayoutAlign + +```typescript +type LayoutAlign = 'start' | 'center' | 'end' | 'stretch'; +``` + +### LayoutJustify + +```typescript +type LayoutJustify = 'start' | 'center' | 'end' | 'space-between'; +``` + +### RendererTheme + +```typescript +type RendererTheme = 'default' | 'light' | 'dark'; +``` + +--- + +## Root Structure + +```typescript +interface UiSchemaRenderer { + version: string; // "2.0" + locale: string; // "zh-CN" + status: UiStatus; + theme: RendererTheme; + + // Protocol-level metadata (not rendered) + meta?: { + requestId?: string; + toolId?: string; + traceId?: string; + userId?: string; + }; + + // Root layout node + root: UiLayoutNode; +} +``` + +--- + +## Node Types + +### Primitive Components + +#### 1. Text Node + +```typescript +interface UiTextNode extends UiBaseNode { + type: 'text'; + content: string; + format: TextFormat; // 'plain' | 'markdown' + role: TextRole; // 'title' | 'subtitle' | 'body' | 'caption' | 'code' + status?: UiStatus; + maxLines?: number; + visible?: boolean; +} +``` + +#### 2. Icon Node + +```typescript +interface UiIconNode extends UiBaseNode { + type: 'icon'; + source: IconSource; // 'icon' | 'emoji' | 'url' + value: string; + color?: string; + size?: number; + visible?: boolean; +} +``` + +#### 3. Badge Node + +```typescript +interface UiBadgeNode extends UiBaseNode { + type: 'badge'; + label: string; + status: UiStatus; + visible?: boolean; +} +``` + +#### 4. Button Node + +```typescript +interface UiButtonNode extends UiBaseNode { + type: 'button'; + label: string; + style: ButtonStyle; + disabled?: boolean; + icon?: UiIconSpec; + action: UiActionPayload; + visible?: boolean; +} +``` + +#### 5. Key-Value Node + +```typescript +interface UiKvNode extends UiBaseNode { + type: 'kv'; + items: UiKvItem[]; + columns?: number; + visible?: boolean; +} + +interface UiKvItem { + key: string; + label?: string; + value: any; + copyable?: boolean; +} +``` + +#### 6. Divider Node + +```typescript +interface UiDividerNode extends UiBaseNode { + type: 'divider'; + inset?: number; + visible?: boolean; +} +``` + +### Layout Containers + +#### 7. Stack Node + +```typescript +interface UiStackNode extends UiBaseNode { + type: 'stack'; + direction: LayoutDirection; // 'vertical' | 'horizontal' + gap?: number; + appearance: LayoutAppearance; // 'plain' | 'card' | 'section' + status?: UiStatus; + align?: LayoutAlign; + justify?: LayoutJustify; + wrap?: boolean; + children: UiNode[]; + visible?: boolean; +} +``` + +#### 8. Grid Node + +```typescript +interface UiGridNode extends UiBaseNode { + type: 'grid'; + columns: number; + gap?: number; + appearance: LayoutAppearance; + status?: UiStatus; + children: UiNode[]; + visible?: boolean; +} +``` + +### Base Node + +```typescript +interface UiBaseNode { + id?: string; + visible?: boolean; +} +``` + +### Node Union + +```typescript +type UiNode = + | UiTextNode + | UiIconNode + | UiBadgeNode + | UiButtonNode + | UiKvNode + | UiDividerNode + | UiStackNode + | UiGridNode; + +type UiLayoutNode = UiStackNode | UiGridNode; +``` + +--- + +## Action Payloads + +```typescript +type UiActionPayload = + | NavigateAction + | UrlAction + | EventAction + | ToolAction + | CopyAction + | PayloadAction; + +// Navigation action +interface NavigateAction { + type: 'navigation'; + path: string; + params?: Record; +} + +// URL action +interface UrlAction { + type: 'url'; + url: string; + target?: '_self' | '_blank'; +} + +// Event action +interface EventAction { + type: 'event'; + event: string; + payload?: Record; +} + +// Tool action +interface ToolAction { + type: 'tool'; + toolId: string; + params?: Record; +} + +// Copy action +interface CopyAction { + type: 'copy'; + content: string; + successMessage?: string; +} + +// Payload action +interface PayloadAction { + type: 'payload'; + payload: Record; + submitTo?: string; +} +``` + +--- + +## Icon Specification + +```typescript +interface UiIconSpec { + source: IconSource; + value: string; + color?: string; + size?: number; +} +``` + +--- + +## Usage Patterns--- + +## JSON Examples + +### Example 1: Simple Text + +```json +{ + "version": "2.0", + "locale": "zh-CN", + "status": "success", + "theme": "default", + "root": { + "type": "stack", + "direction": "vertical", + "gap": 12, + "appearance": "plain", + "children": [ + { + "type": "text", + "content": "操作成功", + "role": "title" + }, + { + "type": "text", + "content": "您的事项已创建完成", + "role": "body" + } + ] + } +} +``` + +### Example 2: Card with Actions + +```json +{ + "version": "2.0", + "locale": "zh-CN", + "status": "success", + "theme": "default", + "root": { + "type": "stack", + "direction": "vertical", + "gap": 16, + "appearance": "card", + "children": [ + { + "type": "text", + "content": "日程已创建", + "role": "title" + }, + { + "type": "kv", + "items": [ + { "key": "title", "label": "主题", "value": "Q1 规划会议" }, + { "key": "time", "label": "时间", "value": "2026-03-15 14:00" } + ], + "columns": 2 + }, + { + "type": "stack", + "direction": "horizontal", + "gap": 8, + "children": [ + { + "type": "button", + "label": "查看详情", + "style": "primary", + "action": { "type": "navigation", "path": "/calendar/evt_abc123" } + }, + { + "type": "button", + "label": "删除", + "style": "danger", + "action": { "type": "tool", "toolId": "calendar.delete", "params": { "eventId": "evt_abc123" } } + } + ] + } + ] + } +} +``` + +### Example 3: Error Status Panel + +```json +{ + "version": "2.0", + "locale": "zh-CN", + "status": "error", + "theme": "default", + "root": { + "type": "stack", + "direction": "vertical", + "gap": 12, + "appearance": "card", + "status": "error", + "children": [ + { + "type": "stack", + "direction": "horizontal", + "gap": 8, + "align": "center", + "justify": "space-between", + "children": [ + { + "type": "text", + "content": "删除失败", + "role": "title" + }, + { + "type": "badge", + "label": "ERROR", + "status": "error" + } + ] + }, + { + "type": "text", + "content": "您没有权限执行此操作", + "role": "body", + "status": "error" + }, + { + "type": "stack", + "direction": "horizontal", + "gap": 8, + "children": [ + { + "type": "button", + "label": "重试", + "style": "primary", + "action": { "type": "tool", "toolId": "user.delete", "params": { "userId": "u1" } } + }, + { + "type": "button", + "label": "联系管理员", + "style": "secondary", + "action": { "type": "url", "url": "mailto:admin@example.com" } + } + ] + } + ] + } +} +``` + +### Example 4: Grid Layout + +```json +{ + "version": "2.0", + "locale": "zh-CN", + "status": "info", + "theme": "default", + "root": { + "type": "grid", + "columns": 3, + "gap": 16, + "appearance": "plain", + "children": [ + { + "type": "card", + "children": [ + { "type": "text", "content": "今日订单", "role": "title" }, + { "type": "text", "content": "128", "role": "subtitle" } + ] + }, + { + "type": "card", + "children": [ + { "type": "text", "content": "待处理", "role": "title" }, + { "type": "text", "content": "24", "role": "subtitle", "status": "warning" } + ] + }, + { + "type": "card", + "children": [ + { "type": "text", "content": "总收入", "role": "title" }, + { "type": "text", "content": "¥8,640", "role": "subtitle", "status": "success" } + ] + } + ] + } +} +``` + +### Example 5: Section Layout + +```json +{ + "version": "2.0", + "locale": "zh-CN", + "status": "success", + "theme": "default", + "root": { + "type": "stack", + "direction": "vertical", + "gap": 24, + "appearance": "plain", + "children": [ + { + "type": "stack", + "direction": "vertical", + "gap": 12, + "appearance": "section", + "children": [ + { "type": "text", "content": "基本信息", "role": "title" }, + { "type": "kv", "items": [ + { "key": "name", "label": "姓名", "value": "张三" }, + { "key": "email", "label": "邮箱", "value": "zhangsan@example.com", "copyable": true } + ]} + ] + }, + { + "type": "stack", + "direction": "vertical", + "gap": 12, + "appearance": "section", + "children": [ + { "type": "text", "content": "账户设置", "role": "title" }, + { "type": "button", "label": "修改密码", "style": "secondary", "action": { "type": "navigation", "path": "/settings/password" } }, + { "type": "button", "label": "退出登录", "style": "ghost", "action": { "type": "event", "event": "logout" } } + ] + } + ] + } +} +``` + +--- + + +--- + +## UiHints (Descriptive UI) + +### Overview + +UiHints is a **descriptive** UI representation designed for AI agents to express UI intent with minimal token cost. It describes **what to show**, not **how to render**. + +The `ui_compiler` transforms UiHints into UiSchemaRenderer for frontend rendering. + +### Design Principles + +1. **Descriptive not Rendered**: Express content intent, not visual instructions +2. **Minimal Token Cost**: Simple structure, semantic field names +3. **Composable**: Supports nested sections and mixed content +4. **Compilable**: Mechanical transformation to UiSchemaRenderer +5. **Lossless**: Main content fields in hints are preserved in renderer + +### Intent Types + +Intent is a **weak hint** - it only affects default layout style, not field presence. + +| Intent | Default Layout | +|--------|----------------| +| `message` | plain | +| `data` | card | +| `list` | plain | +| `status` | card | +| `form` | section | +| `mixed` | card | + +### UiHints Payload + +```typescript +interface UiHintsPayload { + version: string; // "2.1" + intent: UiHintIntent; // Primary display intent (weak hint) + status: UiStatus; // Overall status + + title?: string; // Top-level title + description?: string; // Top-level description + body?: string; // Top-level main body text + bodyFormat?: "plain" | "markdown"; // Body text format + + items?: UiHintKvItem[]; // Top-level key-value items + listItems?: UiHintListItem[]; // Top-level list items + sections?: UiHintSection[]; // Grouped sections + actions?: UiHintAction[]; // Top-level actions + icon?: UiHintIcon; // Top-level icon + meta?: Record; // Extra meta +} + +interface UiHintSection { + title?: string; + description?: string; + icon?: UiHintIcon; + content?: string; + contentFormat?: "plain" | "markdown"; + items?: UiHintKvItem[]; + listItems?: UiHintListItem[]; + actions?: UiHintAction[]; +} + +interface UiHintListItem { + id?: string; + title: string; + subtitle?: string; + description?: string; + icon?: UiHintIcon; + status?: UiHintStatus; + actions?: UiHintAction[]; +} + +interface UiHintKvItem { + key: string; + label?: string; + value?: any; + copyable?: boolean; +} + +interface UiHintAction { + label: string; + style?: "primary" | "secondary" | "ghost" | "danger"; + disabled?: boolean; + action: UiHintActionTarget; +} + +type UiHintActionTarget = + | { type: "navigation"; path: string; params?: Record } + | { type: "url"; url: string; target?: "_self" | "_blank" } + | { type: "event"; event: string; payload?: Record } + | { type: "tool"; toolId: string; params?: Record } + | { type: "copy"; content: string; successMessage?: string } + | { type: "payload"; payload: Record; submitTo?: string }; + +interface UiHintIcon { + source: "icon" | "emoji" | "url"; + value: string; + color?: string; + size?: number; +} +``` + +### Compilation Flow + +``` +Agent Output (UiHints 2.1) + │ + ▼ + ui_compiler + │ + ▼ +UiSchemaRenderer (for frontend rendering) +``` + +### Example + +**Input (UiHints)**: +```json +{ + "intent": "status", + "status": "success", + "title": "日程已创建", + "body": "本次创建已成功完成。", + "items": [ + {"key": "title", "label": "主题", "value": "Q1 规划会议"}, + {"key": "time", "label": "时间", "value": "2026-03-15 14:00"} + ], + "actions": [ + {"label": "查看详情", "style": "primary", "action": {"type": "navigation", "path": "/calendar/evt_123"}}, + {"label": "删除", "style": "danger", "action": {"type": "tool", "toolId": "calendar.delete", "params": {"eventId": "evt_123"}}} + ] +} +``` + +**Output (UiSchemaRenderer)**: +```json +{ + "version": "2.0", + "locale": "zh-CN", + "status": "success", + "theme": "default", + "root": { + "type": "stack", + "appearance": "card", + "status": "success", + "children": [ + { + "type": "stack", + "direction": "horizontal", + "gap": 8, + "children": [ + {"type": "text", "content": "日程已创建", "role": "title"}, + {"type": "badge", "label": "SUCCESS", "status": "success"} + ], + "justify": "space-between", + "align": "center" + }, + {"type": "text", "content": "本次创建已成功完成。", "role": "body"}, + {"type": "kv", "items": [...]}, + { + "type": "stack", + "direction": "horizontal", + "gap": 8, + "children": [ + {"type": "button", "label": "查看详情", "style": "primary", ...}, + {"type": "button", "label": "删除", "style": "danger", ...} + ] + } + ] + } +} +``` + +### Python Implementation + +- `schemas.agent.ui_hints.UiHintsPayload` - Descriptive UI model (v2.1) +- `core.agentscope.runtime.ui_compiler.compile(hints)` - Compile to UiSchemaRenderer