feat: 添加 Agent 步骤事件与图片附件功能

- 新增 stepStarted/stepFinished 事件类型支持
- 前端实现图片附件上传和预览功能
- 后端增强工具结果存储和事件处理
- 完善相关单元测试和集成测试
This commit is contained in:
zl-q
2026-03-12 09:29:57 +08:00
parent 87215f9d41
commit 7b8865e256
45 changed files with 3869 additions and 308 deletions
@@ -1,6 +1,7 @@
import 'dart:async';
import 'dart:convert';
import 'dart:math';
import 'dart:typed_data';
import 'package:dio/dio.dart';
import 'package:image_picker/image_picker.dart';
@@ -80,6 +81,18 @@ class AgUiService {
onEvent(event);
}
Future<Uint8List> fetchAttachmentPreview(String previewPath) async {
final response = await _apiClient.get<List<int>>(
previewPath,
options: Options(responseType: ResponseType.bytes),
);
final payload = response.data;
if (payload is List<int>) {
return Uint8List.fromList(payload);
}
throw StateError('Invalid attachment payload');
}
Future<String> transcribeAudio(String filePath) async {
final formData = FormData.fromMap({
'audio': await MultipartFile.fromFile(
@@ -247,22 +260,27 @@ class AgUiService {
final runId = _nextId(_runIdPrefix);
final contentBlocks = <Map<String, dynamic>>[];
final attachmentMetadata = <Map<String, dynamic>>[];
if (content.isNotEmpty) {
contentBlocks.add({'type': 'text', 'text': content});
}
if (images != null && images.isNotEmpty) {
for (final image in images) {
final bytes = await image.readAsBytes();
final base64 = base64Encode(bytes);
final uploadedAttachments = await _uploadAttachments(
threadId: threadId,
images: images,
);
for (final attachment in uploadedAttachments) {
contentBlocks.add({
'type': 'image',
'source': {
'type': 'base64',
'media_type': 'image/jpeg',
'data': base64,
},
'type': 'binary',
'mimeType': attachment['mimeType'],
'url': attachment['url'],
});
attachmentMetadata.add({
'bucket': attachment['bucket'],
'path': attachment['path'],
'mimeType': attachment['mimeType'],
});
}
}
@@ -286,10 +304,64 @@ class AgUiService {
],
'tools': _buildTools(),
'context': <Map<String, dynamic>>[],
'forwardedProps': <String, dynamic>{},
'forwardedProps': {
if (attachmentMetadata.isNotEmpty) 'attachments': attachmentMetadata,
},
};
}
Future<List<Map<String, dynamic>>> _uploadAttachments({
required String threadId,
required List<XFile> images,
}) async {
final attachments = <Map<String, dynamic>>[];
for (final image in images) {
final mimeType = image.mimeType ?? 'image/jpeg';
final fileBytes = await image.readAsBytes();
final formData = FormData.fromMap({
'threadId': threadId,
'file': MultipartFile.fromBytes(
fileBytes,
filename: image.name,
contentType: DioMediaType.parse(mimeType),
),
});
final response = await _apiClient.post<Map<String, dynamic>>(
'/api/v1/agent/attachments',
data: formData,
);
final payload = response.data;
if (payload is! Map<String, dynamic>) {
throw StateError('Invalid /agent/attachments response');
}
final attachment = payload['attachment'];
if (attachment is! Map<String, dynamic>) {
throw StateError('Missing attachment in /agent/attachments response');
}
final bucket = attachment['bucket'];
final path = attachment['path'];
final uploadedMime = attachment['mimeType'];
final url = attachment['url'];
if (bucket is! String ||
path is! String ||
uploadedMime is! String ||
url is! String ||
bucket.isEmpty ||
path.isEmpty ||
uploadedMime.isEmpty ||
url.isEmpty) {
throw StateError('Invalid attachment reference');
}
attachments.add({
'bucket': bucket,
'path': path,
'mimeType': uploadedMime,
'url': url,
});
}
return attachments;
}
List<Map<String, dynamic>> _buildTools() {
return [
{
@@ -360,6 +432,11 @@ class AgUiService {
'SSE',
_handleMockSse,
);
client.registerHandler(
'/api/v1/agent/attachments',
'POST',
_handleMockUploadAttachment,
);
client.registerHandler(
'/api/v1/agent/transcribe',
'POST',
@@ -371,6 +448,26 @@ class AgUiService {
return {'transcript': '这是模拟语音转写'};
}
Map<String, dynamic> _handleMockUploadAttachment(MockRequest request) {
final payload = request.data;
final threadId = payload is Map<String, dynamic>
? (payload['threadId'] as String?)
: null;
final resolvedThreadId = (threadId != null && threadId.isNotEmpty)
? threadId
: (_threadId ?? _newUuid());
final path =
'agent-inputs/mock/$resolvedThreadId/${_nextId('upload_')}.png';
return {
'attachment': {
'bucket': 'mock-bucket',
'path': path,
'mimeType': 'image/png',
'url': 'https://mock.local/$path',
},
};
}
Map<String, dynamic> _handleMockRun(MockRequest request) {
final payload = request.data;
final runInput = payload is Map<String, dynamic>