qzl/social-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): add voice input capability and standardize tool naming

Browse Source 
- Add voice recording with transcribe endpoint (ASR) for multimodal input
- Android: add RECORD_AUDIO and INTERNET permissions
- Refactor tool naming: frontend tools use 'front.' prefix, backend tools use 'back.'
- Migrate calendar tools: create_calendar_event -> back.mutate/list/delete events
- Add calendar_event_list.v1 and calendar_operation.v1 UI card types
- Update all Flutter and Python tests to match new tool naming conventions
- Add record package dependency for voice recording
This commit is contained in:
zl-q
2026-03-09 00:10:09 +08:00
parent 6c83e35a69
commit 3ac09475ad
30 changed files with 1593 additions and 438 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/lib/features/chat/data/ai/ai_decision_engine.dart
+3 -1
View File
@@ -82,7 +82,9 @@ class AiDecisionEngine {
}
ForceTriggerResult? tryForceTrigger(String text) {
final match = RegExp(r'#tool:(\w+)\s*(\{.*\})?').firstMatch(text);
final match = RegExp(
r'#tool:([A-Za-z0-9_.-]+)\s*(\{.*\})?',
).firstMatch(text);
if (match == null) return null;
final toolName = match.group(1)!;
apps/lib/features/chat/data/models/ag_ui_event.dart
+8
View File
@@ -297,6 +297,14 @@ class ToolCallResultEvent extends AgUiEvent {
if (rawUi is Map<String, dynamic>) {
return UiCard.fromJson(rawUi);
}
final rawResult = payload['result'];
if (rawResult is Map<String, dynamic>) {
final type = rawResult['type'];
final data = rawResult['data'];
if (type is String && data is Map<String, dynamic>) {
return UiCard.fromJson(rawResult);
}
}
return null;
}
apps/lib/features/chat/data/services/ag_ui_service.dart
+84 -114
View File
@@ -2,12 +2,12 @@ import 'dart:async';
import 'dart:convert';
import 'dart:math';
import 'package:dio/dio.dart';
import 'package:social_app/core/api/i_api_client.dart';
import 'package:social_app/core/api/mock_api_client.dart';
import '../ai/ai_decision_engine.dart';
import '../models/ag_ui_event.dart';
import '../models/tool_result.dart';
import '../tools/tool_registry.dart';
import 'mock_history_service.dart';
@@ -36,7 +36,7 @@ class AgUiService {
_decisionEngine = AiDecisionEngine(),
_historyService = MockHistoryService() {
if (_apiClient is MockApiClient) {
_configureMockAgentApi(_apiClient as MockApiClient);
_configureMockAgentApi(_apiClient);
}
}
@@ -77,6 +77,28 @@ class AgUiService {
onEvent(event);
}
Future<String> transcribeAudio(String filePath) async {
final formData = FormData.fromMap({
'audio': await MultipartFile.fromFile(
filePath,
filename: 'recording.wav',
),
});
final response = await _apiClient.post<Map<String, dynamic>>(
'/api/v1/agent/transcribe',
data: formData,
);
final payload = response.data;
if (payload is! Map<String, dynamic>) {
throw StateError('Invalid /agent/transcribe response');
}
final transcript = payload['transcript'];
if (transcript is! String) {
throw StateError('Missing transcript in /agent/transcribe response');
}
return transcript;
}
Future<void> approveToolCall({
required String toolCallId,
required String toolName,
@@ -210,11 +232,7 @@ class AgUiService {
'runId': runId,
'state': <String, dynamic>{},
'messages': [
{
'id': _nextId('user_'),
'role': 'user',
'content': content,
},
{'id': _nextId('user_'), 'role': 'user', 'content': content},
],
'tools': _buildTools(),
'context': <Map<String, dynamic>>[],
@@ -225,33 +243,20 @@ class AgUiService {
List<Map<String, dynamic>> _buildTools() {
return [
{
'name': 'navigate_to_route',
'name': 'front.navigate_to_route',
'description': 'Navigate user to a route in the mobile app.',
'parameters': {
'type': 'object',
'properties': {
'target': {'type': 'string', 'description': 'Route path target'},
'replace': {'type': 'boolean', 'description': 'Use replace navigation'},
'replace': {
'type': 'boolean',
'description': 'Use replace navigation',
},
},
'required': ['target'],
},
},
{
'name': 'create_calendar_event',
'description': 'Create a calendar schedule event.',
'parameters': {
'type': 'object',
'properties': {
'title': {'type': 'string'},
'description': {'type': 'string'},
'startAt': {'type': 'string', 'format': 'date-time'},
'endAt': {'type': 'string', 'format': 'date-time'},
'timezone': {'type': 'string'},
'location': {'type': 'string'},
},
'required': ['title', 'startAt'],
},
},
];
}
@@ -270,7 +275,8 @@ class AgUiService {
return '/api/v1/agent/history?${query.join('&')}';
}
String _nextId(String prefix) => '$prefix${DateTime.now().millisecondsSinceEpoch}';
String _nextId(String prefix) =>
'$prefix${DateTime.now().millisecondsSinceEpoch}';
String _newUuid() {
final random = Random();
@@ -304,6 +310,15 @@ class AgUiService {
'SSE',
_handleMockSse,
);
client.registerHandler(
'/api/v1/agent/transcribe',
'POST',
_handleMockTranscribe,
);
}
Map<String, dynamic> _handleMockTranscribe(MockRequest request) {
return {'transcript': '这是模拟语音转写'};
}
Map<String, dynamic> _handleMockRun(MockRequest request) {
@@ -331,9 +346,9 @@ class AgUiService {
}
Map<String, dynamic> _handleMockResume(MockRequest request) {
final match = RegExp(r'^/api/v1/agent/runs/([^/]+)/resume$').firstMatch(
request.path,
);
final match = RegExp(
r'^/api/v1/agent/runs/([^/]+)/resume$',
).firstMatch(request.path);
final threadId = match?.group(1) ?? (_threadId ?? _newUuid());
final payload = request.data;
final runInput = payload is Map<String, dynamic>
@@ -344,7 +359,11 @@ class AgUiService {
final toolMessage = _extractLatestToolMessage(runInput);
final events = <Map<String, dynamic>>[
{'type': AgUiEventTypeWire.runStarted, 'threadId': threadId, 'runId': runId},
{
'type': AgUiEventTypeWire.runStarted,
'threadId': threadId,
'runId': runId,
},
{
'type': AgUiEventTypeWire.toolCallResult,
'messageId': _nextId(_messageIdPrefix),
@@ -365,7 +384,11 @@ class AgUiService {
'type': AgUiEventTypeWire.textMessageEnd,
'messageId': _nextId(_messageIdPrefix),
},
{'type': AgUiEventTypeWire.runFinished, 'threadId': threadId, 'runId': runId},
{
'type': AgUiEventTypeWire.runFinished,
'threadId': threadId,
'runId': runId,
},
];
_mockSseLinesByThread[threadId] = _toSseLines(events);
return {
@@ -398,7 +421,8 @@ class AgUiService {
final messages = targetDate == null
? <SnapshotMessage>[]
: _historyService.getHistoryForDay(targetDate);
final hasMore = targetDate != null && _historyService.hasEarlierHistory(targetDate);
final hasMore =
targetDate != null && _historyService.hasEarlierHistory(targetDate);
_hasMoreHistory = hasMore;
return {
@@ -421,9 +445,9 @@ class AgUiService {
}
Stream<String> _handleMockSse(MockRequest request) {
final match = RegExp(r'^/api/v1/agent/runs/([^/]+)/events$').firstMatch(
request.path,
);
final match = RegExp(
r'^/api/v1/agent/runs/([^/]+)/events$',
).firstMatch(request.path);
final threadId = match?.group(1);
if (threadId == null) {
return const Stream<String>.empty();
@@ -441,7 +465,11 @@ class AgUiService {
required String userInput,
}) {
final events = <Map<String, dynamic>>[
{'type': AgUiEventTypeWire.runStarted, 'threadId': threadId, 'runId': runId},
{
'type': AgUiEventTypeWire.runStarted,
'threadId': threadId,
'runId': runId,
},
];
final forceTrigger = _decisionEngine.tryForceTrigger(userInput);
@@ -451,19 +479,13 @@ class AgUiService {
toolName = forceTrigger.toolName;
args = forceTrigger.args;
} else if (_looksLikeNavigationIntent(userInput)) {
toolName = 'navigate_to_route';
toolName = 'front.navigate_to_route';
args = {'target': _inferNavigationRoute(userInput), 'replace': false};
} else if (_decisionEngine.shouldTriggerToolCall(userInput)) {
toolName = 'create_calendar_event';
args = _decisionEngine.getToolCallArgs(userInput);
}
if (toolName != null && args != null) {
if (toolName == 'navigate_to_route') {
args = {
...args,
'__nonce': _nextId('nonce_'),
};
if (toolName == 'front.navigate_to_route') {
args = {...args, '__nonce': _nextId('nonce_')};
}
final toolCallId = _nextId(_toolCallIdPrefix);
events.add({
@@ -476,32 +498,20 @@ class AgUiService {
'toolCallId': toolCallId,
'delta': jsonEncode(args),
});
events.add({'type': AgUiEventTypeWire.toolCallEnd, 'toolCallId': toolCallId});
events.add({
'type': AgUiEventTypeWire.toolCallEnd,
'toolCallId': toolCallId,
});
if (toolName == 'navigate_to_route') {
if (toolName == 'front.navigate_to_route') {
// 前端工具：等待审批后由 resume 返回 TOOL_CALL_RESULT。
} else {
final validation = ToolRegistry.validateArgs(toolName, args);
if (!validation.ok) {
events.add({
'type': AgUiEventTypeWire.toolCallError,
'toolCallId': toolCallId,
'error': validation.error ?? 'Validation failed',
'code': 'VALIDATION_ERROR',
});
} else {
final result = _mockCalendarResult(args);
final ui = _buildUiCard(toolName, result);
events.add({
'type': AgUiEventTypeWire.toolCallResult,
'messageId': _nextId(_messageIdPrefix),
'toolCallId': toolCallId,
'content': jsonEncode({
'result': result,
if (ui != null) 'ui': ui.toJson(),
}),
});
}
events.add({
'type': AgUiEventTypeWire.toolCallError,
'toolCallId': toolCallId,
'error': 'Unsupported frontend tool in mock mode',
'code': 'UNSUPPORTED_TOOL',
});
}
}
@@ -518,7 +528,10 @@ class AgUiService {
'messageId': messageId,
'delta': reply,
});
events.add({'type': AgUiEventTypeWire.textMessageEnd, 'messageId': messageId});
events.add({
'type': AgUiEventTypeWire.textMessageEnd,
'messageId': messageId,
});
}
events.add({
@@ -577,57 +590,14 @@ class AgUiService {
if (raw['role'] != 'tool') {
continue;
}
final toolCallId = raw['toolCallId'] as String? ?? _nextId(_toolCallIdPrefix);
final toolCallId =
raw['toolCallId'] as String? ?? _nextId(_toolCallIdPrefix);
final content = raw['content'] as String? ?? '{}';
return (toolCallId, content);
}
return (_nextId(_toolCallIdPrefix), '{}');
}
Map<String, dynamic> _mockCalendarResult(Map<String, dynamic> args) {
final eventId = 'evt_${DateTime.now().millisecondsSinceEpoch}';
return {
'eventId': eventId,
'ok': true,
'message': '日程已创建',
'title': args['title'],
'description': args['description'],
'startAt': args['startAt'],
'endAt': args['endAt'],
'timezone': args['timezone'] ?? 'Asia/Shanghai',
'location': args['location'],
'color': '#4F46E5',
'sourceType': 'agentGenerated',
};
}
UiCard? _buildUiCard(String toolName, Map<String, dynamic> result) {
if (toolName != 'create_calendar_event') {
return null;
}
return UiCard(
cardType: 'calendar_card.v1',
data: CalendarCardData(
id: result['eventId'] ?? '',
title: result['title'] ?? '',
description: result['description'],
startAt: result['startAt'] ?? '',
endAt: result['endAt'],
timezone: result['timezone'],
location: result['location'],
color: result['color'],
sourceType: result['sourceType'],
).toJson(),
actions: [
CardAction(
type: 'link',
label: '查看详情',
target: '/calendar/events/${result['eventId']}',
),
],
);
}
List<String> _generateReplies(String content) {
final intent = _decisionEngine.matchIntent(content);
switch (intent) {
apps/lib/features/chat/data/tools/tool_registry.dart
+1 -58
View File
@@ -4,13 +4,7 @@ typedef ToolHandler =
Future<Map<String, dynamic>> Function(Map<String, dynamic> args);
/// 工具常量
const _toolNameCreateCalendar = 'create_calendar_event';
const _toolNameNavigateRoute = 'navigate_to_route';
const _defaultTimezone = 'Asia/Shanghai';
const _defaultEventColor = '#4F46E5';
const _defaultSourceType = 'agentGenerated';
const _titleMinLength = 1;
const _titleMaxLength = 100;
const _toolNameNavigateRoute = 'front.navigate_to_route';
class ToolDefinition {
final String name;
@@ -33,38 +27,6 @@ class ToolRegistry {
static void initialize() {
if (_initialized) return;
_tools[_toolNameCreateCalendar] = ToolDefinition(
name: _toolNameCreateCalendar,
description: '创建一个日历事件或待办事项',
parameters: {
'type': 'object',
'properties': {
'title': {
'type': 'string',
'description': '事件标题',
'minLength': _titleMinLength,
'maxLength': _titleMaxLength,
},
'description': {'type': 'string', 'description': '事件描述'},
'startAt': {
'type': 'string',
'format': 'date-time',
'description': '开始时间 (ISO8601)',
},
'endAt': {
'type': 'string',
'format': 'date-time',
'description': '结束时间 (ISO8601)',
},
'timezone': {'type': 'string', 'default': _defaultTimezone},
'location': {'type': 'string'},
'notes': {'type': 'string'},
},
'required': ['title', 'startAt'],
},
handler: _handleCreateCalendarEvent,
);
_tools[_toolNameNavigateRoute] = ToolDefinition(
name: _toolNameNavigateRoute,
description: '在前端执行路由跳转',
@@ -82,25 +44,6 @@ class ToolRegistry {
_initialized = true;
}
static Future<Map<String, dynamic>> _handleCreateCalendarEvent(
Map<String, dynamic> args,
) async {
final eventId = 'evt_${DateTime.now().millisecondsSinceEpoch}';
return {
'eventId': eventId,
'ok': true,
'message': '日程已创建',
'title': args['title'],
'description': args['description'],
'startAt': args['startAt'],
'endAt': args['endAt'],
'timezone': args['timezone'] ?? _defaultTimezone,
'location': args['location'],
'color': _defaultEventColor,
'sourceType': _defaultSourceType,
};
}
static Future<Map<String, dynamic>> _handleNavigateRoute(
Map<String, dynamic> args,
) async {
apps/lib/features/chat/presentation/bloc/chat_bloc.dart
+26 -9
View File
@@ -2,11 +2,11 @@ import 'dart:convert';
import 'package:flutter_bloc/flutter_bloc.dart';
import 'package:social_app/core/api/i_api_client.dart';
import 'package:social_app/core/api/mock_api_client.dart';
import 'package:social_app/core/di/injection.dart';
import '../../data/models/ag_ui_event.dart';
import '../../data/models/chat_list_item.dart';
import '../../data/models/tool_result.dart';
import '../../data/services/ag_ui_service.dart';
class ChatState {
@@ -57,7 +57,14 @@ class ChatBloc extends Cubit<ChatState> {
ChatBloc({AgUiService? service, IApiClient? apiClient})
: _service =
service ?? AgUiService(apiClient: apiClient ?? sl<IApiClient>()),
service ??
AgUiService(
apiClient:
apiClient ??
(sl.isRegistered<IApiClient>()
? sl<IApiClient>()
: MockApiClient()),
),
super(const ChatState()) {
_service.onEvent = _handleEvent;
}
@@ -162,13 +169,10 @@ class ChatBloc extends Cubit<ChatState> {
_toolCallArgsBuffer.remove(endEvent.toolCallId);
final updatedItems = state.items.map((item) {
if (item.id == endEvent.toolCallId && item is ToolCallItem) {
final nextStatus = item.toolName == 'navigate_to_route'
final nextStatus = item.toolName == 'front.navigate_to_route'
? ToolCallStatus.pending
: ToolCallStatus.executing;
return item.copyWith(
args: parsedArgs,
status: nextStatus,
);
return item.copyWith(args: parsedArgs, status: nextStatus);
}
return item;
}).toList();
@@ -344,7 +348,10 @@ class ChatBloc extends Cubit<ChatState> {
}
final updatedItems = state.items.map((item) {
if (item is ToolCallItem && item.callId == toolCallId) {
return item.copyWith(status: ToolCallStatus.executing, errorMessage: null);
return item.copyWith(
status: ToolCallStatus.executing,
errorMessage: null,
);
}
return item;
}).toList();
@@ -365,10 +372,20 @@ class ChatBloc extends Cubit<ChatState> {
}
return item;
}).toList();
emit(state.copyWith(items: failedItems, isLoading: false, error: error.toString()));
emit(
state.copyWith(
items: failedItems,
isLoading: false,
error: error.toString(),
),
);
}
}
Future<String> transcribeAudioFile(String filePath) {
return _service.transcribeAudio(filePath);
}
void clearError() {
emit(state.copyWith(error: null));
}
apps/lib/features/chat/ui/widgets/ui_schema_renderer.dart
+105 -1
View File
@@ -4,14 +4,19 @@ import '../../data/models/tool_result.dart';
/// 卡片类型常量
const _calendarCardType = 'calendar_card.v1';
const _calendarListType = 'calendar_event_list.v1';
const _calendarOperationType = 'calendar_operation.v1';
const _errorCardType = 'error_card.v1';
const _aiGeneratedSource = 'ai_generated';
const _agentGeneratedSource = 'agent_generated';
const _primaryActionType = 'primary';
class UiSchemaRenderer {
static Widget render(UiCard card) {
return switch (card.cardType) {
_calendarCardType => _renderCalendarCard(card),
_calendarListType => _renderCalendarList(card),
_calendarOperationType => _renderCalendarOperation(card),
_errorCardType => _renderErrorCard(card),
_ => _renderUnknownCard(card),
};
@@ -22,7 +27,9 @@ class UiSchemaRenderer {
final color = data.color != null
? Color(int.parse(data.color!.replaceFirst('#', '0xFF')))
: AppColors.blue500;
final isAiGenerated = data.sourceType == _aiGeneratedSource;
final isAiGenerated =
data.sourceType == _aiGeneratedSource ||
data.sourceType == _agentGeneratedSource;
return Container(
decoration: BoxDecoration(
@@ -152,6 +159,103 @@ class UiSchemaRenderer {
);
}
static Widget _renderCalendarList(UiCard card) {
final rawItems = card.data['items'];
final items = rawItems is List ? rawItems : const [];
final paginationRaw = card.data['pagination'];
final pagination = paginationRaw is Map<String, dynamic>
? paginationRaw
: const <String, dynamic>{};
final page = pagination['page'];
final total = pagination['total'];
return Container(
padding: EdgeInsets.all(AppSpacing.lg),
decoration: BoxDecoration(
color: AppColors.messageCardBg,
borderRadius: BorderRadius.circular(AppRadius.lg),
border: Border.all(color: AppColors.messageCardBorder),
),
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text(
'日程列表',
style: TextStyle(
fontSize: 16,
fontWeight: FontWeight.w600,
color: AppColors.slate900,
),
),
if (page != null || total != null) ...[
SizedBox(height: AppSpacing.xs),
Text(
'第${page ?? '-'}页 · 共${total ?? '-'}条',
style: TextStyle(fontSize: 12, color: AppColors.slate500),
),
],
SizedBox(height: AppSpacing.sm),
if (items.isEmpty)
Text(
'暂无日程',
style: TextStyle(fontSize: 14, color: AppColors.slate500),
),
for (final item in items)
if (item is Map<String, dynamic>)
Padding(
padding: EdgeInsets.only(bottom: AppSpacing.xs),
child: Text(
item['title']?.toString() ?? '未命名日程',
style: TextStyle(fontSize: 14, color: AppColors.slate700),
),
),
],
),
);
}
static Widget _renderCalendarOperation(UiCard card) {
final ok = card.data['ok'] == true;
final operation = card.data['operation']?.toString() ?? 'operation';
final message = card.data['message']?.toString() ?? (ok ? '操作成功' : '操作失败');
return Container(
padding: EdgeInsets.all(AppSpacing.lg),
decoration: BoxDecoration(
color: ok ? AppColors.messageCardBg : AppColors.warningBackground,
borderRadius: BorderRadius.circular(AppRadius.lg),
border: Border.all(
color: ok ? AppColors.messageCardBorder : AppColors.red400,
),
),
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Text(
'日程$operation结果',
style: TextStyle(
fontSize: 14,
fontWeight: FontWeight.w600,
color: ok ? AppColors.slate900 : AppColors.red600,
),
),
SizedBox(height: AppSpacing.xs),
Text(
message,
style: TextStyle(
fontSize: 13,
color: ok ? AppColors.slate600 : AppColors.red600,
),
),
if (card.actions != null && card.actions!.isNotEmpty) ...[
SizedBox(height: AppSpacing.md),
_buildActions(card.actions!),
],
],
),
);
}
static Widget _renderErrorCard(UiCard card) {
final message = card.data['message'] as String? ?? '发生错误';