qzl/social-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): add voice input capability and standardize tool naming

Browse Source 
- Add voice recording with transcribe endpoint (ASR) for multimodal input
- Android: add RECORD_AUDIO and INTERNET permissions
- Refactor tool naming: frontend tools use 'front.' prefix, backend tools use 'back.'
- Migrate calendar tools: create_calendar_event -> back.mutate/list/delete events
- Add calendar_event_list.v1 and calendar_operation.v1 UI card types
- Update all Flutter and Python tests to match new tool naming conventions
- Add record package dependency for voice recording
This commit is contained in:
zl-q
2026-03-09 00:10:09 +08:00
parent 6c83e35a69
commit 3ac09475ad
30 changed files with 1593 additions and 438 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/test/features/chat/ag_ui_service_test.dart
+36 -36
View File
@@ -26,8 +26,6 @@ class TestableAgUiService extends AgUiService {
final forceTrigger = engine.tryForceTrigger(content);
if (forceTrigger != null) {
await mockToolCallFlowWithArgs(forceTrigger.toolName, forceTrigger.args);
} else if (engine.shouldTriggerToolCall(content)) {
await mockToolCallFlow(content, engine);
}
final replies = generateReplies(content, engine);
@@ -38,13 +36,6 @@ class TestableAgUiService extends AgUiService {
onEvent(RunFinishedEvent(threadId: threadId, runId: runId));
}
Future<void> mockToolCallFlow(String content, AiDecisionEngine engine) async {
final args = engine.getToolCallArgs(content);
if (args == null) return;
await mockToolCallFlowWithArgs('create_calendar_event', args);
}
Future<void> mockToolCallFlowWithArgs(
String toolName,
Map<String, dynamic> args,
@@ -57,6 +48,10 @@ class TestableAgUiService extends AgUiService {
onEvent(ToolCallEndEvent(toolCallId: toolCallId));
if (toolName == 'front.navigate_to_route') {
return;
}
final validation = ToolRegistry.validateArgs(toolName, args);
if (!validation.ok) {
onEvent(
@@ -71,7 +66,7 @@ class TestableAgUiService extends AgUiService {
try {
ToolRegistry.initialize();
final result = await ToolRegistry.execute(toolName, args);
await ToolRegistry.execute(toolName, args);
final messageId = 'msg_${DateTime.now().millisecondsSinceEpoch}';
onEvent(
@@ -157,28 +152,30 @@ void main() {
expect(types.last, AgUiEventType.runFinished);
});
test('creating schedule text triggers tool call events', () async {
await service.sendMessage('提醒我明天10点开会');
test(
'creating schedule text does not trigger frontend tool call events',
() async {
await service.sendMessage('提醒我明天10点开会');
final toolCallStarts = capturedEvents
.whereType<ToolCallStartEvent>()
.toList();
final toolCallEnds = capturedEvents
.whereType<ToolCallEndEvent>()
.toList();
final toolCallResults = capturedEvents
.whereType<ToolCallResultEvent>()
.toList();
final toolCallStarts = capturedEvents
.whereType<ToolCallStartEvent>()
.toList();
final toolCallEnds = capturedEvents
.whereType<ToolCallEndEvent>()
.toList();
final toolCallResults = capturedEvents
.whereType<ToolCallResultEvent>()
.toList();
expect(toolCallStarts.isNotEmpty, true);
expect(toolCallEnds.isNotEmpty, true);
expect(toolCallResults.isNotEmpty, true);
expect(toolCallStarts.first.toolCallName, 'create_calendar_event');
});
expect(toolCallStarts.isEmpty, true);
expect(toolCallEnds.isEmpty, true);
expect(toolCallResults.isEmpty, true);
},
);
test('force trigger with #tool syntax', () async {
await service.sendMessage(
'#tool:create_calendar_event {"title": "Test", "startAt": "2026-03-01T10:00:00Z"}',
'#tool:front.navigate_to_route {"target": "/calendar/dayweek"}',
);
final toolCallStarts = capturedEvents
@@ -186,7 +183,7 @@ void main() {
.toList();
expect(toolCallStarts.isNotEmpty, true);
expect(toolCallStarts.first.toolCallName, 'create_calendar_event');
expect(toolCallStarts.first.toolCallName, 'front.navigate_to_route');
});
test('text message events are emitted for unknown intent', () async {
@@ -215,15 +212,18 @@ void main() {
expect(toolCallStarts.isEmpty, true);
});
test('tool call with invalid args emits error', () async {
await service.sendMessage('#tool:create_calendar_event {}');
test('frontend tool call keeps pending state before approval', () async {
await service.sendMessage('#tool:front.navigate_to_route {}');
final toolCallErrors = capturedEvents
.whereType<ToolCallErrorEvent>()
.toList();
final toolCallStarts = capturedEvents
.whereType<ToolCallStartEvent>()
.toList();
expect(toolCallErrors.isNotEmpty, true);
expect(toolCallErrors.first.error, contains('Missing required fields'));
expect(toolCallStarts.isNotEmpty, true);
expect(toolCallErrors.isEmpty, true);
});
});
@@ -319,7 +319,7 @@ void main() {
await service.sendMessage('初始化会话');
await service.approveToolCall(
toolCallId: 'call-1',
toolName: 'navigate_to_route',
toolName: 'front.navigate_to_route',
args: {
'target': '/calendar/dayweek',
'replace': false,
@@ -349,7 +349,7 @@ void main() {
(e) => e.toolCallId == toolStart.toolCallId,
);
final toolArgs = jsonDecode(toolArgsEvent.delta) as Map<String, dynamic>;
expect(toolStart.toolCallName, 'navigate_to_route');
expect(toolStart.toolCallName, 'front.navigate_to_route');
expect(
events
.whereType<ToolCallResultEvent>()
@@ -360,7 +360,7 @@ void main() {
await realService.approveToolCall(
toolCallId: toolStart.toolCallId,
toolName: 'navigate_to_route',
toolName: 'front.navigate_to_route',
args: toolArgs,
);
@@ -387,7 +387,7 @@ void main() {
expect(
() => realService.approveToolCall(
toolCallId: toolStart.toolCallId,
toolName: 'navigate_to_route',
toolName: 'front.navigate_to_route',
args: toolArgs,
),
throwsA(isA<StateError>()),