feat: 实现 AgentScope ReAct Runner 两阶段执行并重构事件处理
This commit is contained in:
zl-q
@@ -0,0 +1,432 @@
|
||||
# 前后端数据流通指南
|
||||
|
||||
本文档描述前端如何与后端 Agent 系统交互，以及如何渲染后端返回的 UI 数据。
|
||||
|
||||
---
|
||||
|
||||
## 1. 整体交互流程
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ 前端 │
|
||||
├─────────────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌──────────────┐ POST /runs ┌──────────────┐ │
|
||||
│ │ 构造请求 │ ──────────────────▶│ 后端 │ │
|
||||
│ │ (多模态数据) │ │ (任务入队) │ │
|
||||
│ └──────────────┘◀────────────────── └──────────────┘ 202 Accepted │
|
||||
│ │ taskId │
|
||||
│ │ │
|
||||
│ │ GET /runs/{threadId}/events │
|
||||
│ │──────────────────────────────────────────────────────────▶ │
|
||||
│ │ SSE 事件流 ◀───────────────── │
|
||||
│ │ │
|
||||
│ ┌─────┴──────────────────────────────────────────────────────────┐ │
|
||||
│ │ 渲染事件 │ │
|
||||
│ │ - text.delta: 追加文本 │ │
|
||||
│ │ - tool.result: 渲染 toolAgentOutput.ui_hints → UiSchema │ │
|
||||
│ │ - text.end: 渲染 workerAgentOutput.ui_hints → UiSchema │ │
|
||||
│ └────────────────────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ 后端 │
|
||||
├─────────────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ POST /runs │
|
||||
│ → 验证请求 │
|
||||
│ → 任务入队列 (Taskiq) │
|
||||
│ │
|
||||
│ Worker 执行 │
|
||||
│ → emit 事件到 Redis Stream │
|
||||
│ → UiHints → ui_compiler.compile() → UiSchemaRenderer │
|
||||
│ │
|
||||
│ 持久化 │
|
||||
│ → AgentChatMessage (含 uiSchema) │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. 多模态数据处理
|
||||
|
||||
### 2.1 前端上传附件流程
|
||||
|
||||
```javascript
|
||||
// 方式一：先上传附件，获取签名 URL
|
||||
const formData = new FormData();
|
||||
formData.append('threadId', threadId);
|
||||
formData.append('file', imageFile);
|
||||
|
||||
const uploadResponse = await fetch('/api/v1/agent/attachments', {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
});
|
||||
const { attachment } = await uploadResponse.json();
|
||||
// attachment.url 是临时访问 URL
|
||||
|
||||
// 方式二：直接使用已有的签名 URL（如果有）
|
||||
const imageUrl = 'https://storage.example.com/...';
|
||||
```
|
||||
|
||||
### 2.2 构造 RunAgentInput
|
||||
|
||||
```javascript
|
||||
const runInput = {
|
||||
threadId: threadId,
|
||||
runId: `run-${Date.now()}`,
|
||||
state: {},
|
||||
messages: [
|
||||
{
|
||||
id: `msg-${Date.now()}`,
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: '这张图片里有什么？' },
|
||||
{
|
||||
type: 'binary',
|
||||
mimeType: 'image/png',
|
||||
url: attachment.url // 签名 URL
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
tools: [], // 或传入工具定义
|
||||
context: [],
|
||||
forwardedProps: {}
|
||||
};
|
||||
```
|
||||
|
||||
### 2.3 发送请求
|
||||
|
||||
```javascript
|
||||
const response = await fetch('/api/v1/agent/runs', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(runInput)
|
||||
});
|
||||
// 返回 202 Accepted
|
||||
const { taskId, threadId, runId, created } = await response.json();
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. 事件流渲染
|
||||
|
||||
### 3.1 订阅事件
|
||||
|
||||
```javascript
|
||||
class AgentEventHandler {
|
||||
constructor(threadId) {
|
||||
this.eventSource = new EventSource(`/api/v1/agent/runs/${threadId}/events`);
|
||||
this.setupListeners();
|
||||
}
|
||||
|
||||
setupListeners() {
|
||||
this.eventSource.addEventListener('run.started', this.onRunStarted);
|
||||
this.eventSource.addEventListener('step.start', this.onStepStart);
|
||||
this.eventSource.addEventListener('text.delta', this.onTextDelta);
|
||||
this.eventSource.addEventListener('text.message.start', this.onTextStart);
|
||||
this.eventSource.addEventListener('tool.call.start', this.onToolStart);
|
||||
this.eventSource.addEventListener('tool.call.args', this.onToolArgs);
|
||||
this.eventSource.addEventListener('tool.call.result', this.onToolResult);
|
||||
this.eventSource.addEventListener('text.message.end', this.onTextEnd);
|
||||
this.eventSource.addEventListener('run.finished', this.onRunFinished);
|
||||
this.eventSource.addEventListener('run.error', this.onRunError);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3.2 事件处理与渲染
|
||||
|
||||
#### text.delta - 文本增量
|
||||
|
||||
```javascript
|
||||
onTextDelta(event) {
|
||||
const data = JSON.parse(event.data);
|
||||
const { delta, messageId } = data.data;
|
||||
// 追加到对应消息的文本内容
|
||||
appendTextToMessage(messageId, delta);
|
||||
}
|
||||
```
|
||||
|
||||
#### tool.result - 工具结果
|
||||
|
||||
```javascript
|
||||
onToolResult(event) {
|
||||
const data = JSON.parse(event.data);
|
||||
const { toolAgentOutput, toolCallId } = data.data;
|
||||
|
||||
if (toolAgentOutput?.ui_hints) {
|
||||
// 后端返回的是 UiHints，需要编译为 UiSchemaRenderer
|
||||
const uiSchema = compileUiHints(toolAgentOutput.ui_hints);
|
||||
renderUiSchema(uiSchema, toolCallId);
|
||||
} else {
|
||||
// 纯文本结果
|
||||
renderText(toolAgentOutput.result_summary, toolCallId);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### text.end - Worker 完成
|
||||
|
||||
```javascript
|
||||
onTextEnd(event) {
|
||||
const data = JSON.parse(event.data);
|
||||
const { workerAgentOutput, inputTokens, outputTokens, cost } = data.data;
|
||||
|
||||
if (workerAgentOutput?.ui_hints) {
|
||||
// 渲染 UiHints → UiSchemaRenderer
|
||||
const uiSchema = compileUiHints(workerAgentOutput.ui_hints);
|
||||
renderUiSchema(uiSchema, messageId);
|
||||
} else {
|
||||
// 纯文本回复
|
||||
renderText(workerAgentOutput?.answer, messageId);
|
||||
}
|
||||
|
||||
// 显示使用统计
|
||||
showUsageStats({ inputTokens, outputTokens, cost });
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. History 对话历史渲染
|
||||
|
||||
### 4.1 获取历史
|
||||
|
||||
```javascript
|
||||
const response = await fetch(
|
||||
`/api/v1/agent/history?threadId=${threadId}&before=${date}`
|
||||
);
|
||||
const { messages } = await response.json();
|
||||
```
|
||||
|
||||
### 4.2 渲染历史消息
|
||||
|
||||
```javascript
|
||||
messages.forEach(msg => {
|
||||
switch (msg.role) {
|
||||
case 'user':
|
||||
renderUserMessage(msg);
|
||||
break;
|
||||
case 'assistant':
|
||||
if (msg.uiSchema) {
|
||||
// 直接渲染 UiSchemaRenderer（后端已编译好）
|
||||
renderUiSchema(msg.uiSchema);
|
||||
} else {
|
||||
// 纯文本
|
||||
renderText(msg.content);
|
||||
}
|
||||
break;
|
||||
case 'tool':
|
||||
// tool 结果也可能有 uiSchema
|
||||
if (msg.uiSchema) {
|
||||
renderUiSchema(msg.uiSchema);
|
||||
} else {
|
||||
renderText(msg.content);
|
||||
}
|
||||
break;
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. UiSchemaRenderer 渲染
|
||||
|
||||
### 5.1 数据结构
|
||||
|
||||
后端 `ui_compiler.compile()` 输出的结构：
|
||||
|
||||
```typescript
|
||||
interface UiSchemaRenderer {
|
||||
version: "2.0";
|
||||
locale: string;
|
||||
status: "info" | "success" | "warning" | "error" | "pending";
|
||||
theme: "default" | "light" | "dark";
|
||||
meta?: {
|
||||
requestId?: string;
|
||||
toolId?: string;
|
||||
traceId?: string;
|
||||
userId?: string;
|
||||
};
|
||||
root: UiLayoutNode;
|
||||
}
|
||||
|
||||
// 布局节点
|
||||
interface UiStackNode {
|
||||
type: "stack";
|
||||
direction: "vertical" | "horizontal";
|
||||
gap?: number;
|
||||
appearance: "plain" | "card" | "section";
|
||||
status?: UiStatus;
|
||||
align?: LayoutAlign;
|
||||
justify?: LayoutJustify;
|
||||
wrap?: boolean;
|
||||
children: UiNode[];
|
||||
}
|
||||
|
||||
interface UiGridNode {
|
||||
type: "grid";
|
||||
columns: number;
|
||||
gap?: number;
|
||||
appearance: LayoutAppearance;
|
||||
status?: UiStatus;
|
||||
children: UiNode[];
|
||||
}
|
||||
|
||||
// 基础节点
|
||||
type UiNode =
|
||||
| UiTextNode // 文本
|
||||
| UiIconNode // 图标
|
||||
| UiBadgeNode // 标签
|
||||
| UiButtonNode // 按钮
|
||||
| UiKvNode // 键值对
|
||||
| UiDividerNode; // 分割线
|
||||
|
||||
// 文本节点
|
||||
interface UiTextNode {
|
||||
type: "text";
|
||||
content: string;
|
||||
format: "plain" | "markdown";
|
||||
role: "title" | "subtitle" | "body" | "caption" | "code";
|
||||
status?: UiStatus;
|
||||
maxLines?: number;
|
||||
visible?: boolean;
|
||||
}
|
||||
|
||||
// 按钮节点
|
||||
interface UiButtonNode {
|
||||
type: "button";
|
||||
label: string;
|
||||
style: "primary" | "secondary" | "ghost" | "danger";
|
||||
disabled?: boolean;
|
||||
icon?: UiIconSpec;
|
||||
action: UiActionPayload;
|
||||
}
|
||||
|
||||
// 键值对节点
|
||||
interface UiKvNode {
|
||||
type: "kv";
|
||||
items: UiKvItem[];
|
||||
columns?: number;
|
||||
}
|
||||
|
||||
interface UiKvItem {
|
||||
key: string;
|
||||
label?: string;
|
||||
value: any;
|
||||
copyable?: boolean;
|
||||
}
|
||||
|
||||
// Action payloads
|
||||
type UiActionPayload =
|
||||
| { type: "navigation"; path: string; params?: Record<string, any> }
|
||||
| { type: "url"; url: string; target?: "_self" | "_blank" }
|
||||
| { type: "event"; event: string; payload?: Record<string, any> }
|
||||
| { type: "tool"; toolId: string; params?: Record<string, any> }
|
||||
| { type: "copy"; content: string; successMessage?: string }
|
||||
| { type: "payload"; payload: Record<string, any>; submitTo?: string };
|
||||
```
|
||||
|
||||
### 5.2 统一渲染器实现
|
||||
|
||||
```dart
|
||||
class UiSchemaRendererWidget extends StatelessWidget {
|
||||
final UiSchemaRenderer schema;
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return renderNode(schema.root);
|
||||
}
|
||||
|
||||
Widget renderNode(UiNode node) {
|
||||
switch (node.type) {
|
||||
case 'text':
|
||||
return renderTextNode(node as UiTextNode);
|
||||
case 'icon':
|
||||
return renderIconNode(node as UiIconNode);
|
||||
case 'badge':
|
||||
return renderBadgeNode(node as UiBadgeNode);
|
||||
case 'button':
|
||||
return renderButtonNode(node as UiButtonNode);
|
||||
case 'kv':
|
||||
return renderKvNode(node as UiKvNode);
|
||||
case 'divider':
|
||||
return renderDividerNode(node as UiDividerNode);
|
||||
case 'stack':
|
||||
return renderStackNode(node as UiStackNode);
|
||||
case 'grid':
|
||||
return renderGridNode(node as UiGridNode);
|
||||
default:
|
||||
return SizedBox.shrink();
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 5.3 事件渲染 vs History 渲染一致性
|
||||
|
||||
**关键点**：两者使用同一个渲染器
|
||||
|
||||
| 数据来源 | 数据格式 | 处理方式 |
|
||||
|---------|---------|---------|
|
||||
| events text.end | `workerAgentOutput.ui_hints` | compile → UiSchemaRenderer → 渲染 |
|
||||
| events tool.result | `toolAgentOutput.ui_hints` | compile → UiSchemaRenderer → 渲染 |
|
||||
| history assistant | `msg.uiSchema` | 直接渲染 |
|
||||
| history tool | `msg.uiSchema` | 直接渲染 |
|
||||
|
||||
**编译函数**（前端需要实现）：
|
||||
|
||||
```javascript
|
||||
// 将后端 UiHints 编译为 UiSchemaRenderer
|
||||
function compileUiHints(hints) {
|
||||
// 这里可以调用后端的 compile 接口
|
||||
// 或者在前端实现相同的编译逻辑
|
||||
return fetch('/api/v1/agent/compile-ui-hints', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(hints)
|
||||
}).then(r => r.json());
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. UiHints 介绍（可选）
|
||||
|
||||
如果前端需要自己处理 UiHints，以下是结构：
|
||||
|
||||
### UiHintsPayload
|
||||
|
||||
```typescript
|
||||
interface UiHintsPayload {
|
||||
version: "2.1";
|
||||
intent: "message" | "data" | "list" | "status" | "form" | "mixed";
|
||||
status: "info" | "success" | "warning" | "error" | "pending";
|
||||
|
||||
title?: string;
|
||||
description?: string;
|
||||
body?: string;
|
||||
bodyFormat?: "plain" | "markdown";
|
||||
|
||||
items?: UiHintKvItem[];
|
||||
listItems?: UiHintListItem[];
|
||||
sections?: UiHintSection[];
|
||||
actions?: UiHintAction[];
|
||||
icon?: UiHintIcon;
|
||||
meta?: Record<string, any>;
|
||||
}
|
||||
```
|
||||
|
||||
**注意**：History API 返回的消息已经包含了编译好的 `uiSchema`，前端不需要自己编译。
|
||||
|
||||
---
|
||||
|
||||
## 7. 总结
|
||||
|
||||
1. **多模态**：前端上传附件 → 获取签名 URL → 构造 RunAgentInput → POST /runs
|
||||
2. **实时事件**：订阅 /runs/{threadId}/events → 处理 text.delta/tool.result/text.end → 渲染 UiSchema
|
||||
3. **历史消息**：GET /history → 遍历消息 → 有 uiSchema 则渲染，否则渲染纯文本
|
||||
4. **一致性**：events 和 history 使用同一个 UiSchemaRenderer 渲染组件
|
||||
@@ -0,0 +1,747 @@
|
||||
# UI Schema Protocol
|
||||
|
||||
> **NOTE**: This document is the single source of truth. All implementations must follow this specification.
|
||||
|
||||
## Overview
|
||||
|
||||
A generic UI schema for rendering tool/agent execution results. Designed for AI Agent / Tool ecosystem with extensibility.
|
||||
|
||||
**Design Philosophy**: Keep only "primitive components + layout containers". Frontend only needs to recursively render the root layout tree.
|
||||
|
||||
## Version
|
||||
|
||||
- **Current**: `2.0`
|
||||
- **Status**: Active
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ UiSchemaRenderer (root) │
|
||||
│ - version / locale / status / theme │
|
||||
│ - meta (protocol-level metadata) │
|
||||
│ - root (UiLayoutNode - stack or grid) │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Rendering Flow: │
|
||||
│ 1. Backend returns UiSchemaRenderer with root layout │
|
||||
│ 2. Frontend recursively renders root layout tree │
|
||||
│ 3. Layout nodes (stack/grid) contain children │
|
||||
│ 4. Primitive nodes (text/icon/button/etc) are leaves │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Data Types
|
||||
|
||||
### UiStatus
|
||||
|
||||
```typescript
|
||||
type UiStatus = 'info' | 'success' | 'warning' | 'error' | 'pending';
|
||||
```
|
||||
|
||||
### IconSource
|
||||
|
||||
```typescript
|
||||
type IconSource = 'icon' | 'emoji' | 'url';
|
||||
```
|
||||
|
||||
### TextFormat
|
||||
|
||||
```typescript
|
||||
type TextFormat = 'plain' | 'markdown';
|
||||
```
|
||||
|
||||
### TextRole
|
||||
|
||||
```typescript
|
||||
type TextRole = 'title' | 'subtitle' | 'body' | 'caption' | 'code';
|
||||
```
|
||||
|
||||
### ButtonStyle
|
||||
|
||||
```typescript
|
||||
type ButtonStyle = 'primary' | 'secondary' | 'ghost' | 'danger';
|
||||
```
|
||||
|
||||
### LayoutDirection
|
||||
|
||||
```typescript
|
||||
type LayoutDirection = 'vertical' | 'horizontal';
|
||||
```
|
||||
|
||||
### LayoutAppearance
|
||||
|
||||
```typescript
|
||||
type LayoutAppearance = 'plain' | 'card' | 'section';
|
||||
```
|
||||
|
||||
### LayoutAlign
|
||||
|
||||
```typescript
|
||||
type LayoutAlign = 'start' | 'center' | 'end' | 'stretch';
|
||||
```
|
||||
|
||||
### LayoutJustify
|
||||
|
||||
```typescript
|
||||
type LayoutJustify = 'start' | 'center' | 'end' | 'space-between';
|
||||
```
|
||||
|
||||
### RendererTheme
|
||||
|
||||
```typescript
|
||||
type RendererTheme = 'default' | 'light' | 'dark';
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Root Structure
|
||||
|
||||
```typescript
|
||||
interface UiSchemaRenderer {
|
||||
version: string; // "2.0"
|
||||
locale: string; // "zh-CN"
|
||||
status: UiStatus;
|
||||
theme: RendererTheme;
|
||||
|
||||
// Protocol-level metadata (not rendered)
|
||||
meta?: {
|
||||
requestId?: string;
|
||||
toolId?: string;
|
||||
traceId?: string;
|
||||
userId?: string;
|
||||
};
|
||||
|
||||
// Root layout node
|
||||
root: UiLayoutNode;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Node Types
|
||||
|
||||
### Primitive Components
|
||||
|
||||
#### 1. Text Node
|
||||
|
||||
```typescript
|
||||
interface UiTextNode extends UiBaseNode {
|
||||
type: 'text';
|
||||
content: string;
|
||||
format: TextFormat; // 'plain' | 'markdown'
|
||||
role: TextRole; // 'title' | 'subtitle' | 'body' | 'caption' | 'code'
|
||||
status?: UiStatus;
|
||||
maxLines?: number;
|
||||
visible?: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
#### 2. Icon Node
|
||||
|
||||
```typescript
|
||||
interface UiIconNode extends UiBaseNode {
|
||||
type: 'icon';
|
||||
source: IconSource; // 'icon' | 'emoji' | 'url'
|
||||
value: string;
|
||||
color?: string;
|
||||
size?: number;
|
||||
visible?: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
#### 3. Badge Node
|
||||
|
||||
```typescript
|
||||
interface UiBadgeNode extends UiBaseNode {
|
||||
type: 'badge';
|
||||
label: string;
|
||||
status: UiStatus;
|
||||
visible?: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
#### 4. Button Node
|
||||
|
||||
```typescript
|
||||
interface UiButtonNode extends UiBaseNode {
|
||||
type: 'button';
|
||||
label: string;
|
||||
style: ButtonStyle;
|
||||
disabled?: boolean;
|
||||
icon?: UiIconSpec;
|
||||
action: UiActionPayload;
|
||||
visible?: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
#### 5. Key-Value Node
|
||||
|
||||
```typescript
|
||||
interface UiKvNode extends UiBaseNode {
|
||||
type: 'kv';
|
||||
items: UiKvItem[];
|
||||
columns?: number;
|
||||
visible?: boolean;
|
||||
}
|
||||
|
||||
interface UiKvItem {
|
||||
key: string;
|
||||
label?: string;
|
||||
value: any;
|
||||
copyable?: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
#### 6. Divider Node
|
||||
|
||||
```typescript
|
||||
interface UiDividerNode extends UiBaseNode {
|
||||
type: 'divider';
|
||||
inset?: number;
|
||||
visible?: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
### Layout Containers
|
||||
|
||||
#### 7. Stack Node
|
||||
|
||||
```typescript
|
||||
interface UiStackNode extends UiBaseNode {
|
||||
type: 'stack';
|
||||
direction: LayoutDirection; // 'vertical' | 'horizontal'
|
||||
gap?: number;
|
||||
appearance: LayoutAppearance; // 'plain' | 'card' | 'section'
|
||||
status?: UiStatus;
|
||||
align?: LayoutAlign;
|
||||
justify?: LayoutJustify;
|
||||
wrap?: boolean;
|
||||
children: UiNode[];
|
||||
visible?: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
#### 8. Grid Node
|
||||
|
||||
```typescript
|
||||
interface UiGridNode extends UiBaseNode {
|
||||
type: 'grid';
|
||||
columns: number;
|
||||
gap?: number;
|
||||
appearance: LayoutAppearance;
|
||||
status?: UiStatus;
|
||||
children: UiNode[];
|
||||
visible?: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
### Base Node
|
||||
|
||||
```typescript
|
||||
interface UiBaseNode {
|
||||
id?: string;
|
||||
visible?: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
### Node Union
|
||||
|
||||
```typescript
|
||||
type UiNode =
|
||||
| UiTextNode
|
||||
| UiIconNode
|
||||
| UiBadgeNode
|
||||
| UiButtonNode
|
||||
| UiKvNode
|
||||
| UiDividerNode
|
||||
| UiStackNode
|
||||
| UiGridNode;
|
||||
|
||||
type UiLayoutNode = UiStackNode | UiGridNode;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Action Payloads
|
||||
|
||||
```typescript
|
||||
type UiActionPayload =
|
||||
| NavigateAction
|
||||
| UrlAction
|
||||
| EventAction
|
||||
| ToolAction
|
||||
| CopyAction
|
||||
| PayloadAction;
|
||||
|
||||
// Navigation action
|
||||
interface NavigateAction {
|
||||
type: 'navigation';
|
||||
path: string;
|
||||
params?: Record<string, any>;
|
||||
}
|
||||
|
||||
// URL action
|
||||
interface UrlAction {
|
||||
type: 'url';
|
||||
url: string;
|
||||
target?: '_self' | '_blank';
|
||||
}
|
||||
|
||||
// Event action
|
||||
interface EventAction {
|
||||
type: 'event';
|
||||
event: string;
|
||||
payload?: Record<string, any>;
|
||||
}
|
||||
|
||||
// Tool action
|
||||
interface ToolAction {
|
||||
type: 'tool';
|
||||
toolId: string;
|
||||
params?: Record<string, any>;
|
||||
}
|
||||
|
||||
// Copy action
|
||||
interface CopyAction {
|
||||
type: 'copy';
|
||||
content: string;
|
||||
successMessage?: string;
|
||||
}
|
||||
|
||||
// Payload action
|
||||
interface PayloadAction {
|
||||
type: 'payload';
|
||||
payload: Record<string, any>;
|
||||
submitTo?: string;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Icon Specification
|
||||
|
||||
```typescript
|
||||
interface UiIconSpec {
|
||||
source: IconSource;
|
||||
value: string;
|
||||
color?: string;
|
||||
size?: number;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Usage Patterns---
|
||||
|
||||
## JSON Examples
|
||||
|
||||
### Example 1: Simple Text
|
||||
|
||||
```json
|
||||
{
|
||||
"version": "2.0",
|
||||
"locale": "zh-CN",
|
||||
"status": "success",
|
||||
"theme": "default",
|
||||
"root": {
|
||||
"type": "stack",
|
||||
"direction": "vertical",
|
||||
"gap": 12,
|
||||
"appearance": "plain",
|
||||
"children": [
|
||||
{
|
||||
"type": "text",
|
||||
"content": "操作成功",
|
||||
"role": "title"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"content": "您的事项已创建完成",
|
||||
"role": "body"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Example 2: Card with Actions
|
||||
|
||||
```json
|
||||
{
|
||||
"version": "2.0",
|
||||
"locale": "zh-CN",
|
||||
"status": "success",
|
||||
"theme": "default",
|
||||
"root": {
|
||||
"type": "stack",
|
||||
"direction": "vertical",
|
||||
"gap": 16,
|
||||
"appearance": "card",
|
||||
"children": [
|
||||
{
|
||||
"type": "text",
|
||||
"content": "日程已创建",
|
||||
"role": "title"
|
||||
},
|
||||
{
|
||||
"type": "kv",
|
||||
"items": [
|
||||
{ "key": "title", "label": "主题", "value": "Q1 规划会议" },
|
||||
{ "key": "time", "label": "时间", "value": "2026-03-15 14:00" }
|
||||
],
|
||||
"columns": 2
|
||||
},
|
||||
{
|
||||
"type": "stack",
|
||||
"direction": "horizontal",
|
||||
"gap": 8,
|
||||
"children": [
|
||||
{
|
||||
"type": "button",
|
||||
"label": "查看详情",
|
||||
"style": "primary",
|
||||
"action": { "type": "navigation", "path": "/calendar/evt_abc123" }
|
||||
},
|
||||
{
|
||||
"type": "button",
|
||||
"label": "删除",
|
||||
"style": "danger",
|
||||
"action": { "type": "tool", "toolId": "calendar.delete", "params": { "eventId": "evt_abc123" } }
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Example 3: Error Status Panel
|
||||
|
||||
```json
|
||||
{
|
||||
"version": "2.0",
|
||||
"locale": "zh-CN",
|
||||
"status": "error",
|
||||
"theme": "default",
|
||||
"root": {
|
||||
"type": "stack",
|
||||
"direction": "vertical",
|
||||
"gap": 12,
|
||||
"appearance": "card",
|
||||
"status": "error",
|
||||
"children": [
|
||||
{
|
||||
"type": "stack",
|
||||
"direction": "horizontal",
|
||||
"gap": 8,
|
||||
"align": "center",
|
||||
"justify": "space-between",
|
||||
"children": [
|
||||
{
|
||||
"type": "text",
|
||||
"content": "删除失败",
|
||||
"role": "title"
|
||||
},
|
||||
{
|
||||
"type": "badge",
|
||||
"label": "ERROR",
|
||||
"status": "error"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"content": "您没有权限执行此操作",
|
||||
"role": "body",
|
||||
"status": "error"
|
||||
},
|
||||
{
|
||||
"type": "stack",
|
||||
"direction": "horizontal",
|
||||
"gap": 8,
|
||||
"children": [
|
||||
{
|
||||
"type": "button",
|
||||
"label": "重试",
|
||||
"style": "primary",
|
||||
"action": { "type": "tool", "toolId": "user.delete", "params": { "userId": "u1" } }
|
||||
},
|
||||
{
|
||||
"type": "button",
|
||||
"label": "联系管理员",
|
||||
"style": "secondary",
|
||||
"action": { "type": "url", "url": "mailto:admin@example.com" }
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Example 4: Grid Layout
|
||||
|
||||
```json
|
||||
{
|
||||
"version": "2.0",
|
||||
"locale": "zh-CN",
|
||||
"status": "info",
|
||||
"theme": "default",
|
||||
"root": {
|
||||
"type": "grid",
|
||||
"columns": 3,
|
||||
"gap": 16,
|
||||
"appearance": "plain",
|
||||
"children": [
|
||||
{
|
||||
"type": "card",
|
||||
"children": [
|
||||
{ "type": "text", "content": "今日订单", "role": "title" },
|
||||
{ "type": "text", "content": "128", "role": "subtitle" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "card",
|
||||
"children": [
|
||||
{ "type": "text", "content": "待处理", "role": "title" },
|
||||
{ "type": "text", "content": "24", "role": "subtitle", "status": "warning" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "card",
|
||||
"children": [
|
||||
{ "type": "text", "content": "总收入", "role": "title" },
|
||||
{ "type": "text", "content": "¥8,640", "role": "subtitle", "status": "success" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Example 5: Section Layout
|
||||
|
||||
```json
|
||||
{
|
||||
"version": "2.0",
|
||||
"locale": "zh-CN",
|
||||
"status": "success",
|
||||
"theme": "default",
|
||||
"root": {
|
||||
"type": "stack",
|
||||
"direction": "vertical",
|
||||
"gap": 24,
|
||||
"appearance": "plain",
|
||||
"children": [
|
||||
{
|
||||
"type": "stack",
|
||||
"direction": "vertical",
|
||||
"gap": 12,
|
||||
"appearance": "section",
|
||||
"children": [
|
||||
{ "type": "text", "content": "基本信息", "role": "title" },
|
||||
{ "type": "kv", "items": [
|
||||
{ "key": "name", "label": "姓名", "value": "张三" },
|
||||
{ "key": "email", "label": "邮箱", "value": "zhangsan@example.com", "copyable": true }
|
||||
]}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "stack",
|
||||
"direction": "vertical",
|
||||
"gap": 12,
|
||||
"appearance": "section",
|
||||
"children": [
|
||||
{ "type": "text", "content": "账户设置", "role": "title" },
|
||||
{ "type": "button", "label": "修改密码", "style": "secondary", "action": { "type": "navigation", "path": "/settings/password" } },
|
||||
{ "type": "button", "label": "退出登录", "style": "ghost", "action": { "type": "event", "event": "logout" } }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
|
||||
---
|
||||
|
||||
## UiHints (Descriptive UI)
|
||||
|
||||
### Overview
|
||||
|
||||
UiHints is a **descriptive** UI representation designed for AI agents to express UI intent with minimal token cost. It describes **what to show**, not **how to render**.
|
||||
|
||||
The `ui_compiler` transforms UiHints into UiSchemaRenderer for frontend rendering.
|
||||
|
||||
### Design Principles
|
||||
|
||||
1. **Descriptive not Rendered**: Express content intent, not visual instructions
|
||||
2. **Minimal Token Cost**: Simple structure, semantic field names
|
||||
3. **Composable**: Supports nested sections and mixed content
|
||||
4. **Compilable**: Mechanical transformation to UiSchemaRenderer
|
||||
5. **Lossless**: Main content fields in hints are preserved in renderer
|
||||
|
||||
### Intent Types
|
||||
|
||||
Intent is a **weak hint** - it only affects default layout style, not field presence.
|
||||
|
||||
| Intent | Default Layout |
|
||||
|--------|----------------|
|
||||
| `message` | plain |
|
||||
| `data` | card |
|
||||
| `list` | plain |
|
||||
| `status` | card |
|
||||
| `form` | section |
|
||||
| `mixed` | card |
|
||||
|
||||
### UiHints Payload
|
||||
|
||||
```typescript
|
||||
interface UiHintsPayload {
|
||||
version: string; // "2.1"
|
||||
intent: UiHintIntent; // Primary display intent (weak hint)
|
||||
status: UiStatus; // Overall status
|
||||
|
||||
title?: string; // Top-level title
|
||||
description?: string; // Top-level description
|
||||
body?: string; // Top-level main body text
|
||||
bodyFormat?: "plain" | "markdown"; // Body text format
|
||||
|
||||
items?: UiHintKvItem[]; // Top-level key-value items
|
||||
listItems?: UiHintListItem[]; // Top-level list items
|
||||
sections?: UiHintSection[]; // Grouped sections
|
||||
actions?: UiHintAction[]; // Top-level actions
|
||||
icon?: UiHintIcon; // Top-level icon
|
||||
meta?: Record<string, any>; // Extra meta
|
||||
}
|
||||
|
||||
interface UiHintSection {
|
||||
title?: string;
|
||||
description?: string;
|
||||
icon?: UiHintIcon;
|
||||
content?: string;
|
||||
contentFormat?: "plain" | "markdown";
|
||||
items?: UiHintKvItem[];
|
||||
listItems?: UiHintListItem[];
|
||||
actions?: UiHintAction[];
|
||||
}
|
||||
|
||||
interface UiHintListItem {
|
||||
id?: string;
|
||||
title: string;
|
||||
subtitle?: string;
|
||||
description?: string;
|
||||
icon?: UiHintIcon;
|
||||
status?: UiHintStatus;
|
||||
actions?: UiHintAction[];
|
||||
}
|
||||
|
||||
interface UiHintKvItem {
|
||||
key: string;
|
||||
label?: string;
|
||||
value?: any;
|
||||
copyable?: boolean;
|
||||
}
|
||||
|
||||
interface UiHintAction {
|
||||
label: string;
|
||||
style?: "primary" | "secondary" | "ghost" | "danger";
|
||||
disabled?: boolean;
|
||||
action: UiHintActionTarget;
|
||||
}
|
||||
|
||||
type UiHintActionTarget =
|
||||
| { type: "navigation"; path: string; params?: Record<string, any> }
|
||||
| { type: "url"; url: string; target?: "_self" | "_blank" }
|
||||
| { type: "event"; event: string; payload?: Record<string, any> }
|
||||
| { type: "tool"; toolId: string; params?: Record<string, any> }
|
||||
| { type: "copy"; content: string; successMessage?: string }
|
||||
| { type: "payload"; payload: Record<string, any>; submitTo?: string };
|
||||
|
||||
interface UiHintIcon {
|
||||
source: "icon" | "emoji" | "url";
|
||||
value: string;
|
||||
color?: string;
|
||||
size?: number;
|
||||
}
|
||||
```
|
||||
|
||||
### Compilation Flow
|
||||
|
||||
```
|
||||
Agent Output (UiHints 2.1)
|
||||
│
|
||||
▼
|
||||
ui_compiler
|
||||
│
|
||||
▼
|
||||
UiSchemaRenderer (for frontend rendering)
|
||||
```
|
||||
|
||||
### Example
|
||||
|
||||
**Input (UiHints)**:
|
||||
```json
|
||||
{
|
||||
"intent": "status",
|
||||
"status": "success",
|
||||
"title": "日程已创建",
|
||||
"body": "本次创建已成功完成。",
|
||||
"items": [
|
||||
{"key": "title", "label": "主题", "value": "Q1 规划会议"},
|
||||
{"key": "time", "label": "时间", "value": "2026-03-15 14:00"}
|
||||
],
|
||||
"actions": [
|
||||
{"label": "查看详情", "style": "primary", "action": {"type": "navigation", "path": "/calendar/evt_123"}},
|
||||
{"label": "删除", "style": "danger", "action": {"type": "tool", "toolId": "calendar.delete", "params": {"eventId": "evt_123"}}}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Output (UiSchemaRenderer)**:
|
||||
```json
|
||||
{
|
||||
"version": "2.0",
|
||||
"locale": "zh-CN",
|
||||
"status": "success",
|
||||
"theme": "default",
|
||||
"root": {
|
||||
"type": "stack",
|
||||
"appearance": "card",
|
||||
"status": "success",
|
||||
"children": [
|
||||
{
|
||||
"type": "stack",
|
||||
"direction": "horizontal",
|
||||
"gap": 8,
|
||||
"children": [
|
||||
{"type": "text", "content": "日程已创建", "role": "title"},
|
||||
{"type": "badge", "label": "SUCCESS", "status": "success"}
|
||||
],
|
||||
"justify": "space-between",
|
||||
"align": "center"
|
||||
},
|
||||
{"type": "text", "content": "本次创建已成功完成。", "role": "body"},
|
||||
{"type": "kv", "items": [...]},
|
||||
{
|
||||
"type": "stack",
|
||||
"direction": "horizontal",
|
||||
"gap": 8,
|
||||
"children": [
|
||||
{"type": "button", "label": "查看详情", "style": "primary", ...},
|
||||
{"type": "button", "label": "删除", "style": "danger", ...}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Python Implementation
|
||||
|
||||
- `schemas.agent.ui_hints.UiHintsPayload` - Descriptive UI model (v2.1)
|
||||
- `core.agentscope.runtime.ui_compiler.compile(hints)` - Compile to UiSchemaRenderer
|
||||
Reference in New Issue
Block a user