qzl/social-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): add voice input capability and standardize tool naming

Browse Source 
- Add voice recording with transcribe endpoint (ASR) for multimodal input
- Android: add RECORD_AUDIO and INTERNET permissions
- Refactor tool naming: frontend tools use 'front.' prefix, backend tools use 'back.'
- Migrate calendar tools: create_calendar_event -> back.mutate/list/delete events
- Add calendar_event_list.v1 and calendar_operation.v1 UI card types
- Update all Flutter and Python tests to match new tool naming conventions
- Add record package dependency for voice recording
This commit is contained in:
zl-q
2026-03-09 00:10:09 +08:00
parent 6c83e35a69
commit 3ac09475ad
30 changed files with 1593 additions and 438 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/test/features/chat/ag_ui_event_test.dart
+33 -2
View File
@@ -194,7 +194,7 @@ void main() {
final json = {
'type': 'TOOL_CALL_START',
'toolCallId': 'tc_123',
'toolCallName': 'create_calendar_event',
'toolCallName': 'back.mutate_calendar_event',
'parentMessageId': 'msg_001',
};
@@ -203,7 +203,7 @@ void main() {
expect(event, isA<ToolCallStartEvent>());
final toolStart = event as ToolCallStartEvent;
expect(toolStart.toolCallId, 'tc_123');
expect(toolStart.toolCallName, 'create_calendar_event');
expect(toolStart.toolCallName, 'back.mutate_calendar_event');
expect(toolStart.parentMessageId, 'msg_001');
});
@@ -265,6 +265,37 @@ void main() {
expect(toolResult.result['eventId'], 'evt_001');
});
test('ToolCallResultEvent.ui parses from payload.ui', () {
final json = {
'type': 'TOOL_CALL_RESULT',
'messageId': 'msg_123',
'toolCallId': 'tc_123',
'content':
'{"ui":{"type":"calendar_card.v1","version":"v1","data":{"id":"evt_1","title":"会议","startAt":"2026-03-01T10:00:00Z"},"actions":[]}}',
};
final event = AgUiEvent.fromJson(json) as ToolCallResultEvent;
expect(event.ui, isNotNull);
expect(event.ui!.cardType, 'calendar_card.v1');
});
test(
'ToolCallResultEvent.ui parses from payload.result when result is UiCard',
() {
final json = {
'type': 'TOOL_CALL_RESULT',
'messageId': 'msg_123',
'toolCallId': 'tc_123',
'content':
'{"result":{"type":"calendar_operation.v1","version":"v1","data":{"operation":"delete","ok":true},"actions":[]}}',
};
final event = AgUiEvent.fromJson(json) as ToolCallResultEvent;
expect(event.ui, isNotNull);
expect(event.ui!.cardType, 'calendar_operation.v1');
},
);
test('parses ToolCallErrorEvent', () {
final json = {
'type': 'TOOL_CALL_ERROR',
apps/test/features/chat/ag_ui_service_test.dart
+36 -36
View File
@@ -26,8 +26,6 @@ class TestableAgUiService extends AgUiService {
final forceTrigger = engine.tryForceTrigger(content);
if (forceTrigger != null) {
await mockToolCallFlowWithArgs(forceTrigger.toolName, forceTrigger.args);
} else if (engine.shouldTriggerToolCall(content)) {
await mockToolCallFlow(content, engine);
}
final replies = generateReplies(content, engine);
@@ -38,13 +36,6 @@ class TestableAgUiService extends AgUiService {
onEvent(RunFinishedEvent(threadId: threadId, runId: runId));
}
Future<void> mockToolCallFlow(String content, AiDecisionEngine engine) async {
final args = engine.getToolCallArgs(content);
if (args == null) return;
await mockToolCallFlowWithArgs('create_calendar_event', args);
}
Future<void> mockToolCallFlowWithArgs(
String toolName,
Map<String, dynamic> args,
@@ -57,6 +48,10 @@ class TestableAgUiService extends AgUiService {
onEvent(ToolCallEndEvent(toolCallId: toolCallId));
if (toolName == 'front.navigate_to_route') {
return;
}
final validation = ToolRegistry.validateArgs(toolName, args);
if (!validation.ok) {
onEvent(
@@ -71,7 +66,7 @@ class TestableAgUiService extends AgUiService {
try {
ToolRegistry.initialize();
final result = await ToolRegistry.execute(toolName, args);
await ToolRegistry.execute(toolName, args);
final messageId = 'msg_${DateTime.now().millisecondsSinceEpoch}';
onEvent(
@@ -157,28 +152,30 @@ void main() {
expect(types.last, AgUiEventType.runFinished);
});
test('creating schedule text triggers tool call events', () async {
await service.sendMessage('提醒我明天10点开会');
test(
'creating schedule text does not trigger frontend tool call events',
() async {
await service.sendMessage('提醒我明天10点开会');
final toolCallStarts = capturedEvents
.whereType<ToolCallStartEvent>()
.toList();
final toolCallEnds = capturedEvents
.whereType<ToolCallEndEvent>()
.toList();
final toolCallResults = capturedEvents
.whereType<ToolCallResultEvent>()
.toList();
final toolCallStarts = capturedEvents
.whereType<ToolCallStartEvent>()
.toList();
final toolCallEnds = capturedEvents
.whereType<ToolCallEndEvent>()
.toList();
final toolCallResults = capturedEvents
.whereType<ToolCallResultEvent>()
.toList();
expect(toolCallStarts.isNotEmpty, true);
expect(toolCallEnds.isNotEmpty, true);
expect(toolCallResults.isNotEmpty, true);
expect(toolCallStarts.first.toolCallName, 'create_calendar_event');
});
expect(toolCallStarts.isEmpty, true);
expect(toolCallEnds.isEmpty, true);
expect(toolCallResults.isEmpty, true);
},
);
test('force trigger with #tool syntax', () async {
await service.sendMessage(
'#tool:create_calendar_event {"title": "Test", "startAt": "2026-03-01T10:00:00Z"}',
'#tool:front.navigate_to_route {"target": "/calendar/dayweek"}',
);
final toolCallStarts = capturedEvents
@@ -186,7 +183,7 @@ void main() {
.toList();
expect(toolCallStarts.isNotEmpty, true);
expect(toolCallStarts.first.toolCallName, 'create_calendar_event');
expect(toolCallStarts.first.toolCallName, 'front.navigate_to_route');
});
test('text message events are emitted for unknown intent', () async {
@@ -215,15 +212,18 @@ void main() {
expect(toolCallStarts.isEmpty, true);
});
test('tool call with invalid args emits error', () async {
await service.sendMessage('#tool:create_calendar_event {}');
test('frontend tool call keeps pending state before approval', () async {
await service.sendMessage('#tool:front.navigate_to_route {}');
final toolCallErrors = capturedEvents
.whereType<ToolCallErrorEvent>()
.toList();
final toolCallStarts = capturedEvents
.whereType<ToolCallStartEvent>()
.toList();
expect(toolCallErrors.isNotEmpty, true);
expect(toolCallErrors.first.error, contains('Missing required fields'));
expect(toolCallStarts.isNotEmpty, true);
expect(toolCallErrors.isEmpty, true);
});
});
@@ -319,7 +319,7 @@ void main() {
await service.sendMessage('初始化会话');
await service.approveToolCall(
toolCallId: 'call-1',
toolName: 'navigate_to_route',
toolName: 'front.navigate_to_route',
args: {
'target': '/calendar/dayweek',
'replace': false,
@@ -349,7 +349,7 @@ void main() {
(e) => e.toolCallId == toolStart.toolCallId,
);
final toolArgs = jsonDecode(toolArgsEvent.delta) as Map<String, dynamic>;
expect(toolStart.toolCallName, 'navigate_to_route');
expect(toolStart.toolCallName, 'front.navigate_to_route');
expect(
events
.whereType<ToolCallResultEvent>()
@@ -360,7 +360,7 @@ void main() {
await realService.approveToolCall(
toolCallId: toolStart.toolCallId,
toolName: 'navigate_to_route',
toolName: 'front.navigate_to_route',
args: toolArgs,
);
@@ -387,7 +387,7 @@ void main() {
expect(
() => realService.approveToolCall(
toolCallId: toolStart.toolCallId,
toolName: 'navigate_to_route',
toolName: 'front.navigate_to_route',
args: toolArgs,
),
throwsA(isA<StateError>()),
apps/test/features/chat/ai_decision_engine_test.dart
+11 -6
View File
@@ -112,13 +112,18 @@ void main() {
});
group('tryForceTrigger', () {
test('returns ForceTriggerResult for "#tool:create_calendar_event {}"', () {
final result = engine.tryForceTrigger('#tool:create_calendar_event {}');
test(
'returns ForceTriggerResult for "#tool:front.navigate_to_route {}"',
() {
final result = engine.tryForceTrigger(
'#tool:front.navigate_to_route {}',
);
expect(result, isNotNull);
expect(result!.toolName, 'create_calendar_event');
expect(result.args, isEmpty);
});
expect(result, isNotNull);
expect(result!.toolName, 'front.navigate_to_route');
expect(result.args, isEmpty);
},
);
test(
'returns ForceTriggerResult with args for "#tool:custom {"key": "value"}"',
apps/test/features/chat/chat_bloc_test.dart
+38 -3
View File
@@ -194,7 +194,7 @@ void main() {
service.onEvent(
ToolCallStartEvent(
toolCallId: 'tc_1',
toolCallName: 'create_calendar_event',
toolCallName: 'back.mutate_calendar_event',
),
);
},
@@ -203,7 +203,7 @@ void main() {
(s) {
final item = s.items.first;
return item is ToolCallItem &&
item.toolName == 'create_calendar_event' &&
item.toolName == 'back.mutate_calendar_event' &&
item.status == ToolCallStatus.pending;
},
'has pending tool call',
@@ -220,7 +220,7 @@ void main() {
ToolCallItem(
id: 'tc_1',
callId: 'tc_1',
toolName: 'navigate_to_route',
toolName: 'front.navigate_to_route',
args: {'target': '/calendar/dayweek', '__nonce': 'nonce_1'},
status: ToolCallStatus.executing,
timestamp: DateTime.now(),
@@ -241,5 +241,40 @@ void main() {
isA<ChatState>().having((s) => s.items.isEmpty, 'items empty', true),
],
);
blocTest<ChatBloc, ChatState>(
'toolCallResult with ui in payload.result adds ToolResultItem',
build: () => chatBloc,
seed: () => ChatState(
items: [
ToolCallItem(
id: 'tc_2',
callId: 'tc_2',
toolName: 'back.mutate_calendar_event',
args: {'operation': 'create'},
status: ToolCallStatus.executing,
timestamp: DateTime.now(),
sender: MessageSender.ai,
),
],
),
act: (bloc) {
service.onEvent(
ToolCallResultEvent(
messageId: 'msg_tool_2',
toolCallId: 'tc_2',
content:
'{"result":{"type":"calendar_operation.v1","version":"v1","data":{"operation":"delete","ok":true,"message":"done"},"actions":[]}}',
),
);
},
expect: () => [
isA<ChatState>().having(
(s) => s.items.first is ToolResultItem,
'first item is ToolResultItem',
true,
),
],
);
});
}
apps/test/features/chat/tool_registry_test.dart
+31 -62
View File
@@ -12,11 +12,11 @@ void main() {
});
group('getTool', () {
test('returns tool definition for create_calendar_event', () {
final tool = ToolRegistry.getTool('create_calendar_event');
test('returns tool definition for front.navigate_to_route', () {
final tool = ToolRegistry.getTool('front.navigate_to_route');
expect(tool, isNotNull);
expect(tool!.name, 'create_calendar_event');
expect(tool!.name, 'front.navigate_to_route');
expect(tool.description, isNotEmpty);
});
@@ -26,26 +26,16 @@ void main() {
});
group('validateArgs', () {
test('returns error for empty args (missing title)', () {
final result = ToolRegistry.validateArgs('create_calendar_event', {});
test('returns error for empty args (missing target)', () {
final result = ToolRegistry.validateArgs('front.navigate_to_route', {});
expect(result.ok, false);
expect(result.error, contains('title'));
expect(result.error, contains('target'));
});
test('returns error when missing startAt', () {
final result = ToolRegistry.validateArgs('create_calendar_event', {
'title': 'Test Event',
});
expect(result.ok, false);
expect(result.error, contains('startAt'));
});
test('returns ok: true for valid args with title and startAt', () {
final result = ToolRegistry.validateArgs('create_calendar_event', {
'title': 'x',
'startAt': 'x',
test('returns ok: true for valid args', () {
final result = ToolRegistry.validateArgs('front.navigate_to_route', {
'target': '/settings',
});
expect(result.ok, true);
@@ -61,17 +51,6 @@ void main() {
});
group('execute', () {
test('returns eventId on success', () async {
final result = await ToolRegistry.execute('create_calendar_event', {
'title': 'Test Meeting',
'startAt': '2026-03-01T10:00:00Z',
});
expect(result['eventId'], isNotNull);
expect(result['ok'], true);
expect(result['title'], 'Test Meeting');
});
test('throws ToolNotFoundException for unknown tool', () async {
expect(
() => ToolRegistry.execute('unknown_tool', {}),
@@ -79,22 +58,8 @@ void main() {
);
});
test('includes optional fields in result', () async {
final result = await ToolRegistry.execute('create_calendar_event', {
'title': 'Test',
'startAt': '2026-03-01T10:00:00Z',
'description': 'Description',
'location': 'Room A',
'endAt': '2026-03-01T11:00:00Z',
});
expect(result['description'], 'Description');
expect(result['location'], 'Room A');
expect(result['endAt'], '2026-03-01T11:00:00Z');
});
test('navigate_to_route rejects disallowed target', () async {
final result = await ToolRegistry.execute('navigate_to_route', {
test('front.navigate_to_route rejects disallowed target', () async {
final result = await ToolRegistry.execute('front.navigate_to_route', {
'target': '/admin',
});
@@ -102,23 +67,26 @@ void main() {
expect(result['error'], contains('not allowed'));
});
test('navigate_to_route executes allowed target when navigator is bound', () async {
String? navigatedTo;
bool replaced = false;
RouteNavigationTool.instance.bindNavigator((target, {replace = false}) {
navigatedTo = target;
replaced = replace;
});
test(
'front.navigate_to_route executes allowed target when navigator is bound',
() async {
String? navigatedTo;
bool replaced = false;
RouteNavigationTool.instance.bindNavigator((target, {replace = false}) {
navigatedTo = target;
replaced = replace;
});
final result = await ToolRegistry.execute('navigate_to_route', {
'target': '/settings',
'replace': true,
});
final result = await ToolRegistry.execute('front.navigate_to_route', {
'target': '/settings',
'replace': true,
});
expect(result['ok'], true);
expect(navigatedTo, '/settings');
expect(replaced, true);
});
expect(result['ok'], true);
expect(navigatedTo, '/settings');
expect(replaced, true);
},
);
});
group('getAllTools', () {
@@ -126,7 +94,8 @@ void main() {
final tools = ToolRegistry.getAllTools();
expect(tools, isNotEmpty);
expect(tools.any((t) => t.name == 'create_calendar_event'), true);
expect(tools.any((t) => t.name == 'front.navigate_to_route'), true);
expect(tools.any((t) => t.name == 'create_calendar_event'), false);
});
});
}
apps/test/features/chat/ui_schema_renderer_test.dart
+55
View File
@@ -94,6 +94,61 @@ void main() {
expect(find.text('AI生成'), findsOneWidget);
});
testWidgets('calendar_card.v1 renders agent generated tag', (tester) async {
final card = UiCard(
cardType: 'calendar_card.v1',
data: CalendarCardData(
id: 'evt_001',
title: 'Meeting',
startAt: '2026-03-01T10:00:00Z',
sourceType: 'agent_generated',
).toJson(),
);
await tester.pumpWidget(
MaterialApp(home: Scaffold(body: UiSchemaRenderer.render(card))),
);
expect(find.text('AI生成'), findsOneWidget);
});
testWidgets('calendar_event_list.v1 renders list items', (tester) async {
final card = UiCard(
cardType: 'calendar_event_list.v1',
data: {
'items': [
{'id': 'evt_1', 'title': '晨会'},
{'id': 'evt_2', 'title': '评审'},
],
'pagination': {'page': 1, 'total': 2},
},
);
await tester.pumpWidget(
MaterialApp(home: Scaffold(body: UiSchemaRenderer.render(card))),
);
expect(find.text('日程列表'), findsOneWidget);
expect(find.text('晨会'), findsOneWidget);
expect(find.text('评审'), findsOneWidget);
});
testWidgets('calendar_operation.v1 renders operation message', (
tester,
) async {
final card = UiCard(
cardType: 'calendar_operation.v1',
data: {'operation': 'delete', 'ok': true, 'message': '日程已删除'},
);
await tester.pumpWidget(
MaterialApp(home: Scaffold(body: UiSchemaRenderer.render(card))),
);
expect(find.text('日程delete结果'), findsOneWidget);
expect(find.text('日程已删除'), findsOneWidget);
});
testWidgets('error_card.v1 renders error message', (tester) async {
final card = UiCard(
cardType: 'error_card.v1',
apps/test/features/home/ui/screens/home_screen_test.dart
+135 -3
View File
@@ -1,12 +1,36 @@
import 'package:flutter/material.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:lucide_icons/lucide_icons.dart';
import 'package:social_app/core/api/api_exception.dart';
import 'package:social_app/features/home/data/voice_recorder.dart';
import 'package:social_app/features/home/ui/screens/home_screen.dart';
class _FakeVoiceRecorder implements VoiceRecorder {
bool started = false;
String? stoppedPath;
@override
Future<void> start() async {
started = true;
}
@override
Future<String?> stop() async {
started = false;
stoppedPath = '/tmp/test-audio.wav';
return stoppedPath;
}
@override
Future<void> dispose() async {}
}
void main() {
group('HomeScreen Widget Tests', () {
testWidgets('displays input field', (WidgetTester tester) async {
await tester.pumpWidget(const MaterialApp(home: HomeScreen()));
await tester.pumpWidget(
const MaterialApp(home: HomeScreen(autoLoadHistory: false)),
);
await tester.pumpAndSettle();
expect(find.byType(TextField), findsOneWidget);
@@ -14,7 +38,9 @@ void main() {
});
testWidgets('displays header icons', (WidgetTester tester) async {
await tester.pumpWidget(const MaterialApp(home: HomeScreen()));
await tester.pumpWidget(
const MaterialApp(home: HomeScreen(autoLoadHistory: false)),
);
await tester.pumpAndSettle();
expect(find.byIcon(LucideIcons.settings), findsOneWidget);
@@ -25,10 +51,116 @@ void main() {
testWidgets('displays send or mic icon based on input', (
WidgetTester tester,
) async {
await tester.pumpWidget(const MaterialApp(home: HomeScreen()));
await tester.pumpWidget(
const MaterialApp(home: HomeScreen(autoLoadHistory: false)),
);
await tester.pumpAndSettle();
expect(find.byIcon(LucideIcons.mic), findsOneWidget);
});
testWidgets('tap mic starts recording and shows listening state', (
WidgetTester tester,
) async {
final fakeRecorder = _FakeVoiceRecorder();
await tester.pumpWidget(
MaterialApp(
home: HomeScreen(voiceRecorder: fakeRecorder, autoLoadHistory: false),
),
);
await tester.pumpAndSettle();
await tester.tap(find.byIcon(LucideIcons.mic));
await tester.pump();
expect(fakeRecorder.started, true);
expect(find.text('正在聆听...'), findsOneWidget);
expect(find.byIcon(LucideIcons.square), findsOneWidget);
expect(find.byIcon(LucideIcons.send), findsOneWidget);
});
testWidgets('tap send while recording transcribes and auto sends message', (
WidgetTester tester,
) async {
final fakeRecorder = _FakeVoiceRecorder();
String? sentTranscript;
await tester.pumpWidget(
MaterialApp(
home: HomeScreen(
voiceRecorder: fakeRecorder,
autoLoadHistory: false,
onTranscribeAudio: (filePath) async {
expect(filePath, '/tmp/test-audio.wav');
return '语音自动发送';
},
onAutoSendTranscript: (transcript) async {
sentTranscript = transcript;
},
),
),
);
await tester.pumpAndSettle();
await tester.tap(find.byIcon(LucideIcons.mic));
await tester.pump();
await tester.tap(find.byIcon(LucideIcons.send));
await tester.pumpAndSettle();
expect(sentTranscript, '语音自动发送');
expect(find.byIcon(LucideIcons.plus), findsOneWidget);
});
testWidgets('tap stop transcribes audio and fills input', (
WidgetTester tester,
) async {
final fakeRecorder = _FakeVoiceRecorder();
await tester.pumpWidget(
MaterialApp(
home: HomeScreen(
voiceRecorder: fakeRecorder,
autoLoadHistory: false,
onTranscribeAudio: (filePath) async {
expect(filePath, '/tmp/test-audio.wav');
return '语音转文字结果';
},
),
),
);
await tester.pumpAndSettle();
await tester.tap(find.byIcon(LucideIcons.mic));
await tester.pump();
await tester.tap(find.byIcon(LucideIcons.square));
await tester.pumpAndSettle();
expect(find.text('语音转文字结果'), findsOneWidget);
expect(find.byIcon(LucideIcons.plus), findsOneWidget);
});
testWidgets('tap stop shows readable unauthorized message', (
WidgetTester tester,
) async {
final fakeRecorder = _FakeVoiceRecorder();
await tester.pumpWidget(
MaterialApp(
home: HomeScreen(
voiceRecorder: fakeRecorder,
autoLoadHistory: false,
onTranscribeAudio: (_) async {
throw const UnauthorizedException();
},
),
),
);
await tester.pumpAndSettle();
await tester.tap(find.byIcon(LucideIcons.mic));
await tester.pump();
await tester.tap(find.byIcon(LucideIcons.square));
await tester.pumpAndSettle();
expect(find.text('请重新登录'), findsOneWidget);
await tester.pump(const Duration(seconds: 3));
});
});
}