qzl/social-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 添加 Agent 步骤事件与图片附件功能

Browse Source 
- 新增 stepStarted/stepFinished 事件类型支持
- 前端实现图片附件上传和预览功能
- 后端增强工具结果存储和事件处理
- 完善相关单元测试和集成测试
This commit is contained in:
zl-q
2026-03-12 09:29:57 +08:00
parent 87215f9d41
commit 7b8865e256
45 changed files with 3869 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/lib/features/chat/data/models/ag_ui_event.dart
+66 -3
View File
@@ -9,6 +9,8 @@ class AgUiEventTypeWire {
static const runStarted = 'RUN_STARTED';
static const runFinished = 'RUN_FINISHED';
static const runError = 'RUN_ERROR';
static const stepStarted = 'STEP_STARTED';
static const stepFinished = 'STEP_FINISHED';
static const textMessageStart = 'TEXT_MESSAGE_START';
static const textMessageContent = 'TEXT_MESSAGE_CONTENT';
static const textMessageEnd = 'TEXT_MESSAGE_END';
@@ -25,6 +27,8 @@ enum AgUiEventType {
runStarted,
runFinished,
runError,
stepStarted,
stepFinished,
textMessageStart,
textMessageContent,
textMessageEnd,
@@ -43,6 +47,8 @@ const _wireToTypeMap = {
AgUiEventTypeWire.runStarted: AgUiEventType.runStarted,
AgUiEventTypeWire.runFinished: AgUiEventType.runFinished,
AgUiEventTypeWire.runError: AgUiEventType.runError,
AgUiEventTypeWire.stepStarted: AgUiEventType.stepStarted,
AgUiEventTypeWire.stepFinished: AgUiEventType.stepFinished,
AgUiEventTypeWire.textMessageStart: AgUiEventType.textMessageStart,
AgUiEventTypeWire.textMessageContent: AgUiEventType.textMessageContent,
AgUiEventTypeWire.textMessageEnd: AgUiEventType.textMessageEnd,
@@ -60,6 +66,8 @@ const _typeToWireMap = {
AgUiEventType.runStarted: AgUiEventTypeWire.runStarted,
AgUiEventType.runFinished: AgUiEventTypeWire.runFinished,
AgUiEventType.runError: AgUiEventTypeWire.runError,
AgUiEventType.stepStarted: AgUiEventTypeWire.stepStarted,
AgUiEventType.stepFinished: AgUiEventTypeWire.stepFinished,
AgUiEventType.textMessageStart: AgUiEventTypeWire.textMessageStart,
AgUiEventType.textMessageContent: AgUiEventTypeWire.textMessageContent,
AgUiEventType.textMessageEnd: AgUiEventTypeWire.textMessageEnd,
@@ -83,6 +91,8 @@ final _typeToFactory = {
AgUiEventType.runStarted: RunStartedEvent.fromJson,
AgUiEventType.runFinished: RunFinishedEvent.fromJson,
AgUiEventType.runError: RunErrorEvent.fromJson,
AgUiEventType.stepStarted: StepStartedEvent.fromJson,
AgUiEventType.stepFinished: StepFinishedEvent.fromJson,
AgUiEventType.textMessageStart: TextMessageStartEvent.fromJson,
AgUiEventType.textMessageContent: TextMessageContentEvent.fromJson,
AgUiEventType.textMessageEnd: TextMessageEndEvent.fromJson,
@@ -170,6 +180,34 @@ class RunErrorEvent extends AgUiEvent {
Map<String, dynamic> toJson() => _$RunErrorEventToJson(this);
}
@JsonSerializable()
class StepStartedEvent extends AgUiEvent {
final String stepName;
StepStartedEvent({required this.stepName})
: super(type: AgUiEventType.stepStarted);
factory StepStartedEvent.fromJson(Map<String, dynamic> json) =>
_$StepStartedEventFromJson(json);
@override
Map<String, dynamic> toJson() => _$StepStartedEventToJson(this);
}
@JsonSerializable()
class StepFinishedEvent extends AgUiEvent {
final String stepName;
StepFinishedEvent({required this.stepName})
: super(type: AgUiEventType.stepFinished);
factory StepFinishedEvent.fromJson(Map<String, dynamic> json) =>
_$StepFinishedEventFromJson(json);
@override
Map<String, dynamic> toJson() => _$StepFinishedEventToJson(this);
}
@JsonSerializable()
class TextMessageStartEvent extends AgUiEvent {
final String messageId;
@@ -310,10 +348,33 @@ class ToolCallResultEvent extends AgUiEvent {
factory ToolCallResultEvent.fromJson(Map<String, dynamic> json) {
final rawContent = json['content'];
final content = rawContent is String ? rawContent : '';
final hasStructuredFields =
json['ui'] != null || json['result'] != null || json['error'] != null;
final content = switch (rawContent) {
String value when value.trim().startsWith('{') => value,
String value when value.trim().startsWith('[') => value,
String value when hasStructuredFields => jsonEncode({
'toolName': json['toolName'],
'result': json['result'],
'error': json['error'],
'ui': json['ui'],
'content': value,
}),
String value => value,
_ => jsonEncode({
'toolName': json['toolName'],
'result': json['result'],
'error': json['error'],
'ui': json['ui'],
'content': json['content'],
}),
};
final toolCallId =
json['toolCallId'] as String? ?? json['callId'] as String? ?? '';
final messageId = json['messageId'] as String? ?? 'tool-result-$toolCallId';
return ToolCallResultEvent(
messageId: json['messageId'] as String,
toolCallId: json['toolCallId'] as String,
messageId: messageId,
toolCallId: toolCallId,
content: content,
);
}
@@ -388,6 +449,7 @@ class SnapshotMessage {
final String? toolCallId;
final UiCard? ui;
final DateTime? timestamp;
final List<Map<String, dynamic>>? attachments;
SnapshotMessage({
required this.id,
@@ -396,6 +458,7 @@ class SnapshotMessage {
this.toolCallId,
this.ui,
this.timestamp,
this.attachments,
});
factory SnapshotMessage.fromJson(Map<String, dynamic> json) =>
apps/lib/features/chat/data/models/ag_ui_event.g.dart
+18
View File
@@ -14,6 +14,8 @@ const _$AgUiEventTypeEnumMap = {
AgUiEventType.runStarted: 'runStarted',
AgUiEventType.runFinished: 'runFinished',
AgUiEventType.runError: 'runError',
AgUiEventType.stepStarted: 'stepStarted',
AgUiEventType.stepFinished: 'stepFinished',
AgUiEventType.textMessageStart: 'textMessageStart',
AgUiEventType.textMessageContent: 'textMessageContent',
AgUiEventType.textMessageEnd: 'textMessageEnd',
@@ -53,6 +55,18 @@ RunErrorEvent _$RunErrorEventFromJson(Map<String, dynamic> json) =>
Map<String, dynamic> _$RunErrorEventToJson(RunErrorEvent instance) =>
<String, dynamic>{'message': instance.message, 'code': instance.code};
StepStartedEvent _$StepStartedEventFromJson(Map<String, dynamic> json) =>
StepStartedEvent(stepName: json['stepName'] as String);
Map<String, dynamic> _$StepStartedEventToJson(StepStartedEvent instance) =>
<String, dynamic>{'stepName': instance.stepName};
StepFinishedEvent _$StepFinishedEventFromJson(Map<String, dynamic> json) =>
StepFinishedEvent(stepName: json['stepName'] as String);
Map<String, dynamic> _$StepFinishedEventToJson(StepFinishedEvent instance) =>
<String, dynamic>{'stepName': instance.stepName};
TextMessageStartEvent _$TextMessageStartEventFromJson(
Map<String, dynamic> json,
) => TextMessageStartEvent(
@@ -170,6 +184,9 @@ SnapshotMessage _$SnapshotMessageFromJson(Map<String, dynamic> json) =>
timestamp: json['timestamp'] == null
? null
: DateTime.parse(json['timestamp'] as String),
attachments: (json['attachments'] as List<dynamic>?)
?.whereType<Map<String, dynamic>>()
.toList(),
);
Map<String, dynamic> _$SnapshotMessageToJson(SnapshotMessage instance) =>
@@ -180,4 +197,5 @@ Map<String, dynamic> _$SnapshotMessageToJson(SnapshotMessage instance) =>
'toolCallId': instance.toolCallId,
'ui': instance.ui,
'timestamp': instance.timestamp?.toIso8601String(),
'attachments': instance.attachments,
};
apps/lib/features/chat/data/models/chat_list_item.dart
+4
View File
@@ -22,6 +22,7 @@ class TextMessageItem extends ChatListItem {
@override
final MessageSender sender;
final bool isStreaming;
final List<Map<String, dynamic>> attachments;
TextMessageItem({
required this.id,
@@ -29,6 +30,7 @@ class TextMessageItem extends ChatListItem {
required this.timestamp,
required this.sender,
this.isStreaming = false,
this.attachments = const [],
});
@override
@@ -40,12 +42,14 @@ class TextMessageItem extends ChatListItem {
DateTime? timestamp,
MessageSender? sender,
bool? isStreaming,
List<Map<String, dynamic>>? attachments,
}) => TextMessageItem(
id: id ?? this.id,
content: content ?? this.content,
timestamp: timestamp ?? this.timestamp,
sender: sender ?? this.sender,
isStreaming: isStreaming ?? this.isStreaming,
attachments: attachments ?? this.attachments,
);
}
apps/lib/features/chat/data/services/ag_ui_service.dart
+107 -10
View File
@@ -1,6 +1,7 @@
import 'dart:async';
import 'dart:convert';
import 'dart:math';
import 'dart:typed_data';
import 'package:dio/dio.dart';
import 'package:image_picker/image_picker.dart';
@@ -80,6 +81,18 @@ class AgUiService {
onEvent(event);
}
Future<Uint8List> fetchAttachmentPreview(String previewPath) async {
final response = await _apiClient.get<List<int>>(
previewPath,
options: Options(responseType: ResponseType.bytes),
);
final payload = response.data;
if (payload is List<int>) {
return Uint8List.fromList(payload);
}
throw StateError('Invalid attachment payload');
}
Future<String> transcribeAudio(String filePath) async {
final formData = FormData.fromMap({
'audio': await MultipartFile.fromFile(
@@ -247,22 +260,27 @@ class AgUiService {
final runId = _nextId(_runIdPrefix);
final contentBlocks = <Map<String, dynamic>>[];
final attachmentMetadata = <Map<String, dynamic>>[];
if (content.isNotEmpty) {
contentBlocks.add({'type': 'text', 'text': content});
}
if (images != null && images.isNotEmpty) {
for (final image in images) {
final bytes = await image.readAsBytes();
final base64 = base64Encode(bytes);
final uploadedAttachments = await _uploadAttachments(
threadId: threadId,
images: images,
);
for (final attachment in uploadedAttachments) {
contentBlocks.add({
'type': 'image',
'source': {
'type': 'base64',
'media_type': 'image/jpeg',
'data': base64,
},
'type': 'binary',
'mimeType': attachment['mimeType'],
'url': attachment['url'],
});
attachmentMetadata.add({
'bucket': attachment['bucket'],
'path': attachment['path'],
'mimeType': attachment['mimeType'],
});
}
}
@@ -286,10 +304,64 @@ class AgUiService {
],
'tools': _buildTools(),
'context': <Map<String, dynamic>>[],
'forwardedProps': <String, dynamic>{},
'forwardedProps': {
if (attachmentMetadata.isNotEmpty) 'attachments': attachmentMetadata,
},
};
}
Future<List<Map<String, dynamic>>> _uploadAttachments({
required String threadId,
required List<XFile> images,
}) async {
final attachments = <Map<String, dynamic>>[];
for (final image in images) {
final mimeType = image.mimeType ?? 'image/jpeg';
final fileBytes = await image.readAsBytes();
final formData = FormData.fromMap({
'threadId': threadId,
'file': MultipartFile.fromBytes(
fileBytes,
filename: image.name,
contentType: DioMediaType.parse(mimeType),
),
});
final response = await _apiClient.post<Map<String, dynamic>>(
'/api/v1/agent/attachments',
data: formData,
);
final payload = response.data;
if (payload is! Map<String, dynamic>) {
throw StateError('Invalid /agent/attachments response');
}
final attachment = payload['attachment'];
if (attachment is! Map<String, dynamic>) {
throw StateError('Missing attachment in /agent/attachments response');
}
final bucket = attachment['bucket'];
final path = attachment['path'];
final uploadedMime = attachment['mimeType'];
final url = attachment['url'];
if (bucket is! String ||
path is! String ||
uploadedMime is! String ||
url is! String ||
bucket.isEmpty ||
path.isEmpty ||
uploadedMime.isEmpty ||
url.isEmpty) {
throw StateError('Invalid attachment reference');
}
attachments.add({
'bucket': bucket,
'path': path,
'mimeType': uploadedMime,
'url': url,
});
}
return attachments;
}
List<Map<String, dynamic>> _buildTools() {
return [
{
@@ -360,6 +432,11 @@ class AgUiService {
'SSE',
_handleMockSse,
);
client.registerHandler(
'/api/v1/agent/attachments',
'POST',
_handleMockUploadAttachment,
);
client.registerHandler(
'/api/v1/agent/transcribe',
'POST',
@@ -371,6 +448,26 @@ class AgUiService {
return {'transcript': '这是模拟语音转写'};
}
Map<String, dynamic> _handleMockUploadAttachment(MockRequest request) {
final payload = request.data;
final threadId = payload is Map<String, dynamic>
? (payload['threadId'] as String?)
: null;
final resolvedThreadId = (threadId != null && threadId.isNotEmpty)
? threadId
: (_threadId ?? _newUuid());
final path =
'agent-inputs/mock/$resolvedThreadId/${_nextId('upload_')}.png';
return {
'attachment': {
'bucket': 'mock-bucket',
'path': path,
'mimeType': 'image/png',
'url': 'https://mock.local/$path',
},
};
}
Map<String, dynamic> _handleMockRun(MockRequest request) {
final payload = request.data;
final runInput = payload is Map<String, dynamic>