qzl/social-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): add voice input capability and standardize tool naming

Browse Source 
- Add voice recording with transcribe endpoint (ASR) for multimodal input
- Android: add RECORD_AUDIO and INTERNET permissions
- Refactor tool naming: frontend tools use 'front.' prefix, backend tools use 'back.'
- Migrate calendar tools: create_calendar_event -> back.mutate/list/delete events
- Add calendar_event_list.v1 and calendar_operation.v1 UI card types
- Update all Flutter and Python tests to match new tool naming conventions
- Add record package dependency for voice recording
This commit is contained in:
zl-q
2026-03-09 00:10:09 +08:00
parent 6c83e35a69
commit 3ac09475ad
30 changed files with 1593 additions and 438 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/lib/features/chat/data/services/ag_ui_service.dart
+84 -114
View File
@@ -2,12 +2,12 @@ import 'dart:async';
import 'dart:convert';
import 'dart:math';
import 'package:dio/dio.dart';
import 'package:social_app/core/api/i_api_client.dart';
import 'package:social_app/core/api/mock_api_client.dart';
import '../ai/ai_decision_engine.dart';
import '../models/ag_ui_event.dart';
import '../models/tool_result.dart';
import '../tools/tool_registry.dart';
import 'mock_history_service.dart';
@@ -36,7 +36,7 @@ class AgUiService {
_decisionEngine = AiDecisionEngine(),
_historyService = MockHistoryService() {
if (_apiClient is MockApiClient) {
_configureMockAgentApi(_apiClient as MockApiClient);
_configureMockAgentApi(_apiClient);
}
}
@@ -77,6 +77,28 @@ class AgUiService {
onEvent(event);
}
Future<String> transcribeAudio(String filePath) async {
final formData = FormData.fromMap({
'audio': await MultipartFile.fromFile(
filePath,
filename: 'recording.wav',
),
});
final response = await _apiClient.post<Map<String, dynamic>>(
'/api/v1/agent/transcribe',
data: formData,
);
final payload = response.data;
if (payload is! Map<String, dynamic>) {
throw StateError('Invalid /agent/transcribe response');
}
final transcript = payload['transcript'];
if (transcript is! String) {
throw StateError('Missing transcript in /agent/transcribe response');
}
return transcript;
}
Future<void> approveToolCall({
required String toolCallId,
required String toolName,
@@ -210,11 +232,7 @@ class AgUiService {
'runId': runId,
'state': <String, dynamic>{},
'messages': [
{
'id': _nextId('user_'),
'role': 'user',
'content': content,
},
{'id': _nextId('user_'), 'role': 'user', 'content': content},
],
'tools': _buildTools(),
'context': <Map<String, dynamic>>[],
@@ -225,33 +243,20 @@ class AgUiService {
List<Map<String, dynamic>> _buildTools() {
return [
{
'name': 'navigate_to_route',
'name': 'front.navigate_to_route',
'description': 'Navigate user to a route in the mobile app.',
'parameters': {
'type': 'object',
'properties': {
'target': {'type': 'string', 'description': 'Route path target'},
'replace': {'type': 'boolean', 'description': 'Use replace navigation'},
'replace': {
'type': 'boolean',
'description': 'Use replace navigation',
},
},
'required': ['target'],
},
},
{
'name': 'create_calendar_event',
'description': 'Create a calendar schedule event.',
'parameters': {
'type': 'object',
'properties': {
'title': {'type': 'string'},
'description': {'type': 'string'},
'startAt': {'type': 'string', 'format': 'date-time'},
'endAt': {'type': 'string', 'format': 'date-time'},
'timezone': {'type': 'string'},
'location': {'type': 'string'},
},
'required': ['title', 'startAt'],
},
},
];
}
@@ -270,7 +275,8 @@ class AgUiService {
return '/api/v1/agent/history?${query.join('&')}';
}
String _nextId(String prefix) => '$prefix${DateTime.now().millisecondsSinceEpoch}';
String _nextId(String prefix) =>
'$prefix${DateTime.now().millisecondsSinceEpoch}';
String _newUuid() {
final random = Random();
@@ -304,6 +310,15 @@ class AgUiService {
'SSE',
_handleMockSse,
);
client.registerHandler(
'/api/v1/agent/transcribe',
'POST',
_handleMockTranscribe,
);
}
Map<String, dynamic> _handleMockTranscribe(MockRequest request) {
return {'transcript': '这是模拟语音转写'};
}
Map<String, dynamic> _handleMockRun(MockRequest request) {
@@ -331,9 +346,9 @@ class AgUiService {
}
Map<String, dynamic> _handleMockResume(MockRequest request) {
final match = RegExp(r'^/api/v1/agent/runs/([^/]+)/resume$').firstMatch(
request.path,
);
final match = RegExp(
r'^/api/v1/agent/runs/([^/]+)/resume$',
).firstMatch(request.path);
final threadId = match?.group(1) ?? (_threadId ?? _newUuid());
final payload = request.data;
final runInput = payload is Map<String, dynamic>
@@ -344,7 +359,11 @@ class AgUiService {
final toolMessage = _extractLatestToolMessage(runInput);
final events = <Map<String, dynamic>>[
{'type': AgUiEventTypeWire.runStarted, 'threadId': threadId, 'runId': runId},
{
'type': AgUiEventTypeWire.runStarted,
'threadId': threadId,
'runId': runId,
},
{
'type': AgUiEventTypeWire.toolCallResult,
'messageId': _nextId(_messageIdPrefix),
@@ -365,7 +384,11 @@ class AgUiService {
'type': AgUiEventTypeWire.textMessageEnd,
'messageId': _nextId(_messageIdPrefix),
},
{'type': AgUiEventTypeWire.runFinished, 'threadId': threadId, 'runId': runId},
{
'type': AgUiEventTypeWire.runFinished,
'threadId': threadId,
'runId': runId,
},
];
_mockSseLinesByThread[threadId] = _toSseLines(events);
return {
@@ -398,7 +421,8 @@ class AgUiService {
final messages = targetDate == null
? <SnapshotMessage>[]
: _historyService.getHistoryForDay(targetDate);
final hasMore = targetDate != null && _historyService.hasEarlierHistory(targetDate);
final hasMore =
targetDate != null && _historyService.hasEarlierHistory(targetDate);
_hasMoreHistory = hasMore;
return {
@@ -421,9 +445,9 @@ class AgUiService {
}
Stream<String> _handleMockSse(MockRequest request) {
final match = RegExp(r'^/api/v1/agent/runs/([^/]+)/events$').firstMatch(
request.path,
);
final match = RegExp(
r'^/api/v1/agent/runs/([^/]+)/events$',
).firstMatch(request.path);
final threadId = match?.group(1);
if (threadId == null) {
return const Stream<String>.empty();
@@ -441,7 +465,11 @@ class AgUiService {
required String userInput,
}) {
final events = <Map<String, dynamic>>[
{'type': AgUiEventTypeWire.runStarted, 'threadId': threadId, 'runId': runId},
{
'type': AgUiEventTypeWire.runStarted,
'threadId': threadId,
'runId': runId,
},
];
final forceTrigger = _decisionEngine.tryForceTrigger(userInput);
@@ -451,19 +479,13 @@ class AgUiService {
toolName = forceTrigger.toolName;
args = forceTrigger.args;
} else if (_looksLikeNavigationIntent(userInput)) {
toolName = 'navigate_to_route';
toolName = 'front.navigate_to_route';
args = {'target': _inferNavigationRoute(userInput), 'replace': false};
} else if (_decisionEngine.shouldTriggerToolCall(userInput)) {
toolName = 'create_calendar_event';
args = _decisionEngine.getToolCallArgs(userInput);
}
if (toolName != null && args != null) {
if (toolName == 'navigate_to_route') {
args = {
...args,
'__nonce': _nextId('nonce_'),
};
if (toolName == 'front.navigate_to_route') {
args = {...args, '__nonce': _nextId('nonce_')};
}
final toolCallId = _nextId(_toolCallIdPrefix);
events.add({
@@ -476,32 +498,20 @@ class AgUiService {
'toolCallId': toolCallId,
'delta': jsonEncode(args),
});
events.add({'type': AgUiEventTypeWire.toolCallEnd, 'toolCallId': toolCallId});
events.add({
'type': AgUiEventTypeWire.toolCallEnd,
'toolCallId': toolCallId,
});
if (toolName == 'navigate_to_route') {
if (toolName == 'front.navigate_to_route') {
// 前端工具：等待审批后由 resume 返回 TOOL_CALL_RESULT。
} else {
final validation = ToolRegistry.validateArgs(toolName, args);
if (!validation.ok) {
events.add({
'type': AgUiEventTypeWire.toolCallError,
'toolCallId': toolCallId,
'error': validation.error ?? 'Validation failed',
'code': 'VALIDATION_ERROR',
});
} else {
final result = _mockCalendarResult(args);
final ui = _buildUiCard(toolName, result);
events.add({
'type': AgUiEventTypeWire.toolCallResult,
'messageId': _nextId(_messageIdPrefix),
'toolCallId': toolCallId,
'content': jsonEncode({
'result': result,
if (ui != null) 'ui': ui.toJson(),
}),
});
}
events.add({
'type': AgUiEventTypeWire.toolCallError,
'toolCallId': toolCallId,
'error': 'Unsupported frontend tool in mock mode',
'code': 'UNSUPPORTED_TOOL',
});
}
}
@@ -518,7 +528,10 @@ class AgUiService {
'messageId': messageId,
'delta': reply,
});
events.add({'type': AgUiEventTypeWire.textMessageEnd, 'messageId': messageId});
events.add({
'type': AgUiEventTypeWire.textMessageEnd,
'messageId': messageId,
});
}
events.add({
@@ -577,57 +590,14 @@ class AgUiService {
if (raw['role'] != 'tool') {
continue;
}
final toolCallId = raw['toolCallId'] as String? ?? _nextId(_toolCallIdPrefix);
final toolCallId =
raw['toolCallId'] as String? ?? _nextId(_toolCallIdPrefix);
final content = raw['content'] as String? ?? '{}';
return (toolCallId, content);
}
return (_nextId(_toolCallIdPrefix), '{}');
}
Map<String, dynamic> _mockCalendarResult(Map<String, dynamic> args) {
final eventId = 'evt_${DateTime.now().millisecondsSinceEpoch}';
return {
'eventId': eventId,
'ok': true,
'message': '日程已创建',
'title': args['title'],
'description': args['description'],
'startAt': args['startAt'],
'endAt': args['endAt'],
'timezone': args['timezone'] ?? 'Asia/Shanghai',
'location': args['location'],
'color': '#4F46E5',
'sourceType': 'agentGenerated',
};
}
UiCard? _buildUiCard(String toolName, Map<String, dynamic> result) {
if (toolName != 'create_calendar_event') {
return null;
}
return UiCard(
cardType: 'calendar_card.v1',
data: CalendarCardData(
id: result['eventId'] ?? '',
title: result['title'] ?? '',
description: result['description'],
startAt: result['startAt'] ?? '',
endAt: result['endAt'],
timezone: result['timezone'],
location: result['location'],
color: result['color'],
sourceType: result['sourceType'],
).toJson(),
actions: [
CardAction(
type: 'link',
label: '查看详情',
target: '/calendar/events/${result['eventId']}',
),
],
);
}
List<String> _generateReplies(String content) {
final intent = _decisionEngine.matchIntent(content);
switch (intent) {