feat: 添加 Agent 步骤事件与图片附件功能

- 新增 stepStarted/stepFinished 事件类型支持 - 前端实现图片附件上传和预览功能 - 后端增强工具结果存储和事件处理 - 完善相关单元测试和集成测试
2026-03-12 09:29:57 +08:00
parent 87215f9d41
commit 7b8865e256
45 changed files with 3869 additions and 308 deletions
@@ -9,6 +9,8 @@ class AgUiEventTypeWire {
  static const runStarted = 'RUN_STARTED';
  static const runFinished = 'RUN_FINISHED';
  static const runError = 'RUN_ERROR';
+  static const stepStarted = 'STEP_STARTED';
+  static const stepFinished = 'STEP_FINISHED';
  static const textMessageStart = 'TEXT_MESSAGE_START';
  static const textMessageContent = 'TEXT_MESSAGE_CONTENT';
  static const textMessageEnd = 'TEXT_MESSAGE_END';
@@ -25,6 +27,8 @@ enum AgUiEventType {
  runStarted,
  runFinished,
  runError,
+  stepStarted,
+  stepFinished,
  textMessageStart,
  textMessageContent,
  textMessageEnd,
@@ -43,6 +47,8 @@ const _wireToTypeMap = {
  AgUiEventTypeWire.runStarted: AgUiEventType.runStarted,
  AgUiEventTypeWire.runFinished: AgUiEventType.runFinished,
  AgUiEventTypeWire.runError: AgUiEventType.runError,
+  AgUiEventTypeWire.stepStarted: AgUiEventType.stepStarted,
+  AgUiEventTypeWire.stepFinished: AgUiEventType.stepFinished,
  AgUiEventTypeWire.textMessageStart: AgUiEventType.textMessageStart,
  AgUiEventTypeWire.textMessageContent: AgUiEventType.textMessageContent,
  AgUiEventTypeWire.textMessageEnd: AgUiEventType.textMessageEnd,
@@ -60,6 +66,8 @@ const _typeToWireMap = {
  AgUiEventType.runStarted: AgUiEventTypeWire.runStarted,
  AgUiEventType.runFinished: AgUiEventTypeWire.runFinished,
  AgUiEventType.runError: AgUiEventTypeWire.runError,
+  AgUiEventType.stepStarted: AgUiEventTypeWire.stepStarted,
+  AgUiEventType.stepFinished: AgUiEventTypeWire.stepFinished,
  AgUiEventType.textMessageStart: AgUiEventTypeWire.textMessageStart,
  AgUiEventType.textMessageContent: AgUiEventTypeWire.textMessageContent,
  AgUiEventType.textMessageEnd: AgUiEventTypeWire.textMessageEnd,
@@ -83,6 +91,8 @@ final _typeToFactory = {
  AgUiEventType.runStarted: RunStartedEvent.fromJson,
  AgUiEventType.runFinished: RunFinishedEvent.fromJson,
  AgUiEventType.runError: RunErrorEvent.fromJson,
+  AgUiEventType.stepStarted: StepStartedEvent.fromJson,
+  AgUiEventType.stepFinished: StepFinishedEvent.fromJson,
  AgUiEventType.textMessageStart: TextMessageStartEvent.fromJson,
  AgUiEventType.textMessageContent: TextMessageContentEvent.fromJson,
  AgUiEventType.textMessageEnd: TextMessageEndEvent.fromJson,
@@ -170,6 +180,34 @@ class RunErrorEvent extends AgUiEvent {
  Map<String, dynamic> toJson() => _$RunErrorEventToJson(this);
 }

+@JsonSerializable()
+class StepStartedEvent extends AgUiEvent {
+  final String stepName;
+
+  StepStartedEvent({required this.stepName})
+    : super(type: AgUiEventType.stepStarted);
+
+  factory StepStartedEvent.fromJson(Map<String, dynamic> json) =>
+      _$StepStartedEventFromJson(json);
+
+  @override
+  Map<String, dynamic> toJson() => _$StepStartedEventToJson(this);
+}
+
+@JsonSerializable()
+class StepFinishedEvent extends AgUiEvent {
+  final String stepName;
+
+  StepFinishedEvent({required this.stepName})
+    : super(type: AgUiEventType.stepFinished);
+
+  factory StepFinishedEvent.fromJson(Map<String, dynamic> json) =>
+      _$StepFinishedEventFromJson(json);
+
+  @override
+  Map<String, dynamic> toJson() => _$StepFinishedEventToJson(this);
+}
+
@JsonSerializable()
 class TextMessageStartEvent extends AgUiEvent {
  final String messageId;
@@ -310,10 +348,33 @@ class ToolCallResultEvent extends AgUiEvent {

  factory ToolCallResultEvent.fromJson(Map<String, dynamic> json) {
    final rawContent = json['content'];
-    final content = rawContent is String ? rawContent : '';
+    final hasStructuredFields =
+        json['ui'] != null || json['result'] != null || json['error'] != null;
+    final content = switch (rawContent) {
+      String value when value.trim().startsWith('{') => value,
+      String value when value.trim().startsWith('[') => value,
+      String value when hasStructuredFields => jsonEncode({
+        'toolName': json['toolName'],
+        'result': json['result'],
+        'error': json['error'],
+        'ui': json['ui'],
+        'content': value,
+      }),
+      String value => value,
+      _ => jsonEncode({
+        'toolName': json['toolName'],
+        'result': json['result'],
+        'error': json['error'],
+        'ui': json['ui'],
+        'content': json['content'],
+      }),
+    };
+    final toolCallId =
+        json['toolCallId'] as String? ?? json['callId'] as String? ?? '';
+    final messageId = json['messageId'] as String? ?? 'tool-result-$toolCallId';
    return ToolCallResultEvent(
-      messageId: json['messageId'] as String,
-      toolCallId: json['toolCallId'] as String,
+      messageId: messageId,
+      toolCallId: toolCallId,
      content: content,
    );
  }
@@ -388,6 +449,7 @@ class SnapshotMessage {
  final String? toolCallId;
  final UiCard? ui;
  final DateTime? timestamp;
+  final List<Map<String, dynamic>>? attachments;

  SnapshotMessage({
    required this.id,
@@ -396,6 +458,7 @@ class SnapshotMessage {
    this.toolCallId,
    this.ui,
    this.timestamp,
+    this.attachments,
  });

  factory SnapshotMessage.fromJson(Map<String, dynamic> json) =>
@@ -14,6 +14,8 @@ const _$AgUiEventTypeEnumMap = {
  AgUiEventType.runStarted: 'runStarted',
  AgUiEventType.runFinished: 'runFinished',
  AgUiEventType.runError: 'runError',
+  AgUiEventType.stepStarted: 'stepStarted',
+  AgUiEventType.stepFinished: 'stepFinished',
  AgUiEventType.textMessageStart: 'textMessageStart',
  AgUiEventType.textMessageContent: 'textMessageContent',
  AgUiEventType.textMessageEnd: 'textMessageEnd',
@@ -53,6 +55,18 @@ RunErrorEvent _$RunErrorEventFromJson(Map<String, dynamic> json) =>
 Map<String, dynamic> _$RunErrorEventToJson(RunErrorEvent instance) =>
    <String, dynamic>{'message': instance.message, 'code': instance.code};

+StepStartedEvent _$StepStartedEventFromJson(Map<String, dynamic> json) =>
+    StepStartedEvent(stepName: json['stepName'] as String);
+
+Map<String, dynamic> _$StepStartedEventToJson(StepStartedEvent instance) =>
+    <String, dynamic>{'stepName': instance.stepName};
+
+StepFinishedEvent _$StepFinishedEventFromJson(Map<String, dynamic> json) =>
+    StepFinishedEvent(stepName: json['stepName'] as String);
+
+Map<String, dynamic> _$StepFinishedEventToJson(StepFinishedEvent instance) =>
+    <String, dynamic>{'stepName': instance.stepName};
+
 TextMessageStartEvent _$TextMessageStartEventFromJson(
  Map<String, dynamic> json,
 ) => TextMessageStartEvent(
@@ -170,6 +184,9 @@ SnapshotMessage _$SnapshotMessageFromJson(Map<String, dynamic> json) =>
      timestamp: json['timestamp'] == null
          ? null
          : DateTime.parse(json['timestamp'] as String),
+      attachments: (json['attachments'] as List<dynamic>?)
+          ?.whereType<Map<String, dynamic>>()
+          .toList(),
    );

 Map<String, dynamic> _$SnapshotMessageToJson(SnapshotMessage instance) =>
@@ -180,4 +197,5 @@ Map<String, dynamic> _$SnapshotMessageToJson(SnapshotMessage instance) =>
      'toolCallId': instance.toolCallId,
      'ui': instance.ui,
      'timestamp': instance.timestamp?.toIso8601String(),
+      'attachments': instance.attachments,
    };
@@ -22,6 +22,7 @@ class TextMessageItem extends ChatListItem {
  @override
  final MessageSender sender;
  final bool isStreaming;
+  final List<Map<String, dynamic>> attachments;

  TextMessageItem({
    required this.id,
@@ -29,6 +30,7 @@ class TextMessageItem extends ChatListItem {
    required this.timestamp,
    required this.sender,
    this.isStreaming = false,
+    this.attachments = const [],
  });

  @override
@@ -40,12 +42,14 @@ class TextMessageItem extends ChatListItem {
    DateTime? timestamp,
    MessageSender? sender,
    bool? isStreaming,
+    List<Map<String, dynamic>>? attachments,
  }) => TextMessageItem(
    id: id ?? this.id,
    content: content ?? this.content,
    timestamp: timestamp ?? this.timestamp,
    sender: sender ?? this.sender,
    isStreaming: isStreaming ?? this.isStreaming,
+    attachments: attachments ?? this.attachments,
  );
 }

@@ -1,6 +1,7 @@
 import 'dart:async';
 import 'dart:convert';
 import 'dart:math';
+import 'dart:typed_data';

 import 'package:dio/dio.dart';
 import 'package:image_picker/image_picker.dart';
@@ -80,6 +81,18 @@ class AgUiService {
    onEvent(event);
  }

+  Future<Uint8List> fetchAttachmentPreview(String previewPath) async {
+    final response = await _apiClient.get<List<int>>(
+      previewPath,
+      options: Options(responseType: ResponseType.bytes),
+    );
+    final payload = response.data;
+    if (payload is List<int>) {
+      return Uint8List.fromList(payload);
+    }
+    throw StateError('Invalid attachment payload');
+  }
+
  Future<String> transcribeAudio(String filePath) async {
    final formData = FormData.fromMap({
      'audio': await MultipartFile.fromFile(
@@ -247,22 +260,27 @@ class AgUiService {
    final runId = _nextId(_runIdPrefix);

    final contentBlocks = <Map<String, dynamic>>[];
+    final attachmentMetadata = <Map<String, dynamic>>[];

    if (content.isNotEmpty) {
      contentBlocks.add({'type': 'text', 'text': content});
    }

    if (images != null && images.isNotEmpty) {
-      for (final image in images) {
-        final bytes = await image.readAsBytes();
-        final base64 = base64Encode(bytes);
+      final uploadedAttachments = await _uploadAttachments(
+        threadId: threadId,
+        images: images,
+      );
+      for (final attachment in uploadedAttachments) {
        contentBlocks.add({
-          'type': 'image',
-          'source': {
-            'type': 'base64',
-            'media_type': 'image/jpeg',
-            'data': base64,
-          },
+          'type': 'binary',
+          'mimeType': attachment['mimeType'],
+          'url': attachment['url'],
+        });
+        attachmentMetadata.add({
+          'bucket': attachment['bucket'],
+          'path': attachment['path'],
+          'mimeType': attachment['mimeType'],
        });
      }
    }
@@ -286,10 +304,64 @@ class AgUiService {
      ],
      'tools': _buildTools(),
      'context': <Map<String, dynamic>>[],
-      'forwardedProps': <String, dynamic>{},
+      'forwardedProps': {
+        if (attachmentMetadata.isNotEmpty) 'attachments': attachmentMetadata,
+      },
    };
  }

+  Future<List<Map<String, dynamic>>> _uploadAttachments({
+    required String threadId,
+    required List<XFile> images,
+  }) async {
+    final attachments = <Map<String, dynamic>>[];
+    for (final image in images) {
+      final mimeType = image.mimeType ?? 'image/jpeg';
+      final fileBytes = await image.readAsBytes();
+      final formData = FormData.fromMap({
+        'threadId': threadId,
+        'file': MultipartFile.fromBytes(
+          fileBytes,
+          filename: image.name,
+          contentType: DioMediaType.parse(mimeType),
+        ),
+      });
+      final response = await _apiClient.post<Map<String, dynamic>>(
+        '/api/v1/agent/attachments',
+        data: formData,
+      );
+      final payload = response.data;
+      if (payload is! Map<String, dynamic>) {
+        throw StateError('Invalid /agent/attachments response');
+      }
+      final attachment = payload['attachment'];
+      if (attachment is! Map<String, dynamic>) {
+        throw StateError('Missing attachment in /agent/attachments response');
+      }
+      final bucket = attachment['bucket'];
+      final path = attachment['path'];
+      final uploadedMime = attachment['mimeType'];
+      final url = attachment['url'];
+      if (bucket is! String ||
+          path is! String ||
+          uploadedMime is! String ||
+          url is! String ||
+          bucket.isEmpty ||
+          path.isEmpty ||
+          uploadedMime.isEmpty ||
+          url.isEmpty) {
+        throw StateError('Invalid attachment reference');
+      }
+      attachments.add({
+        'bucket': bucket,
+        'path': path,
+        'mimeType': uploadedMime,
+        'url': url,
+      });
+    }
+    return attachments;
+  }
+
  List<Map<String, dynamic>> _buildTools() {
    return [
      {
@@ -360,6 +432,11 @@ class AgUiService {
      'SSE',
      _handleMockSse,
    );
+    client.registerHandler(
+      '/api/v1/agent/attachments',
+      'POST',
+      _handleMockUploadAttachment,
+    );
    client.registerHandler(
      '/api/v1/agent/transcribe',
      'POST',
@@ -371,6 +448,26 @@ class AgUiService {
    return {'transcript': '这是模拟语音转写'};
  }

+  Map<String, dynamic> _handleMockUploadAttachment(MockRequest request) {
+    final payload = request.data;
+    final threadId = payload is Map<String, dynamic>
+        ? (payload['threadId'] as String?)
+        : null;
+    final resolvedThreadId = (threadId != null && threadId.isNotEmpty)
+        ? threadId
+        : (_threadId ?? _newUuid());
+    final path =
+        'agent-inputs/mock/$resolvedThreadId/${_nextId('upload_')}.png';
+    return {
+      'attachment': {
+        'bucket': 'mock-bucket',
+        'path': path,
+        'mimeType': 'image/png',
+        'url': 'https://mock.local/$path',
+      },
+    };
+  }
+
  Map<String, dynamic> _handleMockRun(MockRequest request) {
    final payload = request.data;
    final runInput = payload is Map<String, dynamic>
@@ -1,4 +1,5 @@
 import 'dart:convert';
+import 'dart:typed_data';

 import 'package:flutter_bloc/flutter_bloc.dart';
 import 'package:image_picker/image_picker.dart';
@@ -10,6 +11,8 @@ import '../../data/models/ag_ui_event.dart';
 import '../../data/models/chat_list_item.dart';
 import '../../data/services/ag_ui_service.dart';

+enum AgentStage { intent, execution, report }
+
 class ChatState {
  final List<ChatListItem> items;
  final bool isSending;
@@ -21,6 +24,7 @@ class ChatState {
  final String? error;
  final DateTime? oldestLoadedDate;
  final bool hasEarlierHistory;
+  final AgentStage? currentStage;

  const ChatState({
    this.items = const [],
@@ -33,6 +37,7 @@ class ChatState {
    this.error,
    this.oldestLoadedDate,
    this.hasEarlierHistory = false,
+    this.currentStage,
  });

  bool get isLoading =>
@@ -55,6 +60,7 @@ class ChatState {
    Object? error = _unset,
    Object? oldestLoadedDate = _unset,
    bool? hasEarlierHistory,
+    Object? currentStage = _unset,
  }) {
    return ChatState(
      items: items ?? this.items,
@@ -71,6 +77,9 @@ class ChatState {
          ? this.oldestLoadedDate
          : oldestLoadedDate as DateTime?,
      hasEarlierHistory: hasEarlierHistory ?? this.hasEarlierHistory,
+      currentStage: currentStage == _unset
+          ? this.currentStage
+          : currentStage as AgentStage?,
    );
  }
 }
@@ -78,6 +87,9 @@ class ChatState {
 class ChatBloc extends Cubit<ChatState> {
  final AgUiService _service;
  final Map<String, String> _toolCallArgsBuffer = {};
+  final Map<String, Uint8List> _attachmentPreviewCache = <String, Uint8List>{};
+  final Map<String, Future<Uint8List?>> _attachmentPreviewInflight =
+      <String, Future<Uint8List?>>{};

  ChatBloc({AgUiService? service, IApiClient? apiClient})
    : _service =
@@ -102,6 +114,7 @@ class ChatBloc extends Cubit<ChatState> {
            isWaitingFirstToken: true,
            isCancelling: false,
            error: null,
+            currentStage: null,
          ),
        );
      case AgUiEventType.runFinished:
@@ -112,6 +125,7 @@ class ChatBloc extends Cubit<ChatState> {
            isStreaming: false,
            isCancelling: false,
            currentMessageId: null,
+            currentStage: null,
          ),
        );
      case AgUiEventType.runError:
@@ -124,8 +138,13 @@ class ChatBloc extends Cubit<ChatState> {
            isCancelling: false,
            currentMessageId: null,
            error: errorEvent.message,
+            currentStage: null,
          ),
        );
+      case AgUiEventType.stepStarted:
+        _handleStepStarted(event as StepStartedEvent);
+      case AgUiEventType.stepFinished:
+        _handleStepFinished(event as StepFinishedEvent);
      case AgUiEventType.textMessageStart:
        _handleTextMessageStart(event as TextMessageStartEvent);
      case AgUiEventType.textMessageContent:
@@ -151,6 +170,16 @@ class ChatBloc extends Cubit<ChatState> {
    }
  }

+  void _handleStepStarted(StepStartedEvent event) {
+    emit(state.copyWith(currentStage: _stageFromName(event.stepName)));
+  }
+
+  void _handleStepFinished(StepFinishedEvent event) {
+    if (state.currentStage == _stageFromName(event.stepName)) {
+      emit(state.copyWith(currentStage: null));
+    }
+  }
+
  void _handleTextMessageStart(TextMessageStartEvent startEvent) {
    final newMessage = TextMessageItem(
      id: startEvent.messageId,
@@ -327,6 +356,7 @@ class ChatBloc extends Cubit<ChatState> {
            content: msg.content ?? '',
            timestamp: timestamp,
            sender: MessageSender.user,
+            attachments: msg.attachments ?? const [],
          );
        case 'assistant':
          return TextMessageItem(
@@ -369,11 +399,20 @@ class ChatBloc extends Cubit<ChatState> {
  }

  Future<void> sendMessage(String content, {List<XFile>? images}) async {
+    final attachments = (images ?? const <XFile>[])
+        .map(
+          (image) => <String, dynamic>{
+            "path": image.path,
+            "mimeType": "image/*",
+          },
+        )
+        .toList();
    final userMessage = TextMessageItem(
      id: 'user-${DateTime.now().millisecondsSinceEpoch}',
      content: content,
      timestamp: DateTime.now(),
      sender: MessageSender.user,
+      attachments: attachments,
    );
    emit(
      state.copyWith(
@@ -509,7 +548,43 @@ class ChatBloc extends Cubit<ChatState> {
    }
  }

+  Future<Uint8List?> loadAttachmentPreview(String previewPath) async {
+    final cached = _attachmentPreviewCache[previewPath];
+    if (cached != null) {
+      return cached;
+    }
+    final pending = _attachmentPreviewInflight[previewPath];
+    if (pending != null) {
+      return pending;
+    }
+    final future = _service
+        .fetchAttachmentPreview(previewPath)
+        .then((bytes) {
+          _attachmentPreviewCache[previewPath] = bytes;
+          return bytes;
+        })
+        .catchError((_) => null)
+        .whenComplete(() {
+          _attachmentPreviewInflight.remove(previewPath);
+        });
+    _attachmentPreviewInflight[previewPath] = future;
+    return future;
+  }
+
  void clearError() {
    emit(state.copyWith(error: null));
  }
 }
+
+AgentStage? _stageFromName(String value) {
+  switch (value) {
+    case 'intent':
+      return AgentStage.intent;
+    case 'execution':
+      return AgentStage.execution;
+    case 'report':
+      return AgentStage.report;
+    default:
+      return null;
+  }
+}