feat(agent): add voice input capability and standardize tool naming
- Add voice recording with transcribe endpoint (ASR) for multimodal input - Android: add RECORD_AUDIO and INTERNET permissions - Refactor tool naming: frontend tools use 'front.' prefix, backend tools use 'back.' - Migrate calendar tools: create_calendar_event -> back.mutate/list/delete events - Add calendar_event_list.v1 and calendar_operation.v1 UI card types - Update all Flutter and Python tests to match new tool naming conventions - Add record package dependency for voice recording
This commit is contained in:
zl-q
@@ -1,11 +1,15 @@
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:flutter_bloc/flutter_bloc.dart';
|
||||
import 'package:go_router/go_router.dart';
|
||||
import 'package:lucide_icons/lucide_icons.dart';
|
||||
import '../../../../core/api/api_exception.dart';
|
||||
import '../../../../core/theme/design_tokens.dart';
|
||||
import '../../../chat/data/models/chat_list_item.dart';
|
||||
import '../../../chat/presentation/bloc/chat_bloc.dart';
|
||||
import '../../../chat/data/tools/route_navigation_tool.dart';
|
||||
import '../../data/voice_recorder.dart';
|
||||
import '../../../chat/ui/widgets/ui_schema_renderer.dart';
|
||||
import '../../../../shared/widgets/toast/toast.dart';
|
||||
import '../../../../shared/widgets/toast/toast_type.dart';
|
||||
@@ -23,22 +27,42 @@ const _cornerRadius = 12.0;
|
||||
const _inputMinHeight = 48.0;
|
||||
const _inputRadius = 24.0;
|
||||
const _scrollDurationMs = 300;
|
||||
const _rippleDurationMs = 1200;
|
||||
const _recordingDotSize = 10.0;
|
||||
|
||||
/// 颜色常量
|
||||
const _chatBgColor = Color(0xFFF8FAFC);
|
||||
const _userBubbleColor = Color(0xFFEAF1FB);
|
||||
|
||||
class HomeScreen extends StatefulWidget {
|
||||
const HomeScreen({super.key});
|
||||
final VoiceRecorder? voiceRecorder;
|
||||
final Future<String> Function(String filePath)? onTranscribeAudio;
|
||||
final Future<void> Function(String transcript)? onAutoSendTranscript;
|
||||
final bool autoLoadHistory;
|
||||
|
||||
const HomeScreen({
|
||||
super.key,
|
||||
this.voiceRecorder,
|
||||
this.onTranscribeAudio,
|
||||
this.onAutoSendTranscript,
|
||||
this.autoLoadHistory = true,
|
||||
});
|
||||
|
||||
@override
|
||||
State<HomeScreen> createState() => _HomeScreenState();
|
||||
}
|
||||
|
||||
class _HomeScreenState extends State<HomeScreen> {
|
||||
class _HomeScreenState extends State<HomeScreen>
|
||||
with SingleTickerProviderStateMixin {
|
||||
final TextEditingController _messageController = TextEditingController();
|
||||
final ScrollController _scrollController = ScrollController();
|
||||
late final ChatBloc _chatBloc;
|
||||
late final VoiceRecorder _voiceRecorder;
|
||||
late final Future<String> Function(String filePath) _transcribeAudio;
|
||||
late final Future<void> Function(String transcript) _autoSendTranscript;
|
||||
late final AnimationController _listeningAnimationController;
|
||||
bool _isRecording = false;
|
||||
bool _isTranscribing = false;
|
||||
|
||||
bool get _hasMessage => _messageController.text.trim().isNotEmpty;
|
||||
|
||||
@@ -47,7 +71,17 @@ class _HomeScreenState extends State<HomeScreen> {
|
||||
super.initState();
|
||||
_messageController.addListener(_onMessageChanged);
|
||||
_chatBloc = ChatBloc();
|
||||
_chatBloc.loadHistory();
|
||||
_voiceRecorder = widget.voiceRecorder ?? RecordVoiceRecorder();
|
||||
_transcribeAudio =
|
||||
widget.onTranscribeAudio ?? _chatBloc.transcribeAudioFile;
|
||||
_autoSendTranscript = widget.onAutoSendTranscript ?? _chatBloc.sendMessage;
|
||||
_listeningAnimationController = AnimationController(
|
||||
vsync: this,
|
||||
duration: const Duration(milliseconds: _rippleDurationMs),
|
||||
);
|
||||
if (widget.autoLoadHistory) {
|
||||
_chatBloc.loadHistory();
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
@@ -55,6 +89,8 @@ class _HomeScreenState extends State<HomeScreen> {
|
||||
_messageController.removeListener(_onMessageChanged);
|
||||
_messageController.dispose();
|
||||
_scrollController.dispose();
|
||||
_listeningAnimationController.dispose();
|
||||
_voiceRecorder.dispose();
|
||||
_chatBloc.close();
|
||||
RouteNavigationTool.instance.clearNavigator();
|
||||
super.dispose();
|
||||
@@ -341,7 +377,7 @@ class _HomeScreenState extends State<HomeScreen> {
|
||||
),
|
||||
const SizedBox(width: 8),
|
||||
Text(statusText, style: TextStyle(fontSize: 12, color: statusColor)),
|
||||
if (item.toolName == 'navigate_to_route' &&
|
||||
if (item.toolName == 'front.navigate_to_route' &&
|
||||
item.status == ToolCallStatus.pending) ...[
|
||||
const SizedBox(width: 8),
|
||||
GestureDetector(
|
||||
@@ -376,7 +412,9 @@ class _HomeScreenState extends State<HomeScreen> {
|
||||
crossAxisAlignment: CrossAxisAlignment.center,
|
||||
children: [
|
||||
GestureDetector(
|
||||
onTap: () => _showBottomSheet(context),
|
||||
onTap: _isRecording
|
||||
? _stopRecording
|
||||
: () => _showBottomSheet(context),
|
||||
child: Container(
|
||||
width: 36,
|
||||
height: 36,
|
||||
@@ -385,10 +423,10 @@ class _HomeScreenState extends State<HomeScreen> {
|
||||
shape: BoxShape.circle,
|
||||
border: Border.all(color: AppColors.slate300),
|
||||
),
|
||||
child: const Icon(
|
||||
LucideIcons.plus,
|
||||
child: Icon(
|
||||
_isRecording ? LucideIcons.square : LucideIcons.plus,
|
||||
size: 20,
|
||||
color: AppColors.slate500,
|
||||
color: _isRecording ? AppColors.red600 : AppColors.slate500,
|
||||
),
|
||||
),
|
||||
),
|
||||
@@ -406,32 +444,42 @@ class _HomeScreenState extends State<HomeScreen> {
|
||||
crossAxisAlignment: CrossAxisAlignment.center,
|
||||
children: [
|
||||
Expanded(
|
||||
child: TextField(
|
||||
controller: _messageController,
|
||||
minLines: 1,
|
||||
maxLines: 3,
|
||||
decoration: const InputDecoration(
|
||||
hintText: '输入消息...',
|
||||
border: InputBorder.none,
|
||||
enabledBorder: InputBorder.none,
|
||||
focusedBorder: InputBorder.none,
|
||||
disabledBorder: InputBorder.none,
|
||||
errorBorder: InputBorder.none,
|
||||
focusedErrorBorder: InputBorder.none,
|
||||
isDense: true,
|
||||
contentPadding: EdgeInsets.zero,
|
||||
filled: false,
|
||||
),
|
||||
onSubmitted: (_) => _sendMessage(context),
|
||||
),
|
||||
child: _isRecording
|
||||
? _buildListeningIndicator()
|
||||
: TextField(
|
||||
controller: _messageController,
|
||||
minLines: 1,
|
||||
maxLines: 3,
|
||||
decoration: const InputDecoration(
|
||||
hintText: '输入消息...',
|
||||
border: InputBorder.none,
|
||||
enabledBorder: InputBorder.none,
|
||||
focusedBorder: InputBorder.none,
|
||||
disabledBorder: InputBorder.none,
|
||||
errorBorder: InputBorder.none,
|
||||
focusedErrorBorder: InputBorder.none,
|
||||
isDense: true,
|
||||
contentPadding: EdgeInsets.zero,
|
||||
filled: false,
|
||||
),
|
||||
onSubmitted: (_) => _sendMessage(context),
|
||||
),
|
||||
),
|
||||
const SizedBox(width: 8),
|
||||
GestureDetector(
|
||||
onTap: _hasMessage ? () => _sendMessage(context) : null,
|
||||
onTap: _isTranscribing
|
||||
? null
|
||||
: _isRecording
|
||||
? () => _stopRecording(autoSendAfterTranscribe: true)
|
||||
: _hasMessage
|
||||
? () => _sendMessage(context)
|
||||
: _startRecording,
|
||||
child: Icon(
|
||||
_hasMessage ? LucideIcons.send : LucideIcons.mic,
|
||||
_isRecording || _hasMessage
|
||||
? LucideIcons.send
|
||||
: LucideIcons.mic,
|
||||
size: _iconSize,
|
||||
color: _hasMessage
|
||||
color: _isRecording || _hasMessage
|
||||
? AppColors.blue600
|
||||
: AppColors.slate500,
|
||||
),
|
||||
@@ -462,6 +510,134 @@ class _HomeScreenState extends State<HomeScreen> {
|
||||
});
|
||||
}
|
||||
|
||||
Widget _buildListeningIndicator() {
|
||||
return SizedBox(
|
||||
height: _inputMinHeight,
|
||||
child: Row(
|
||||
crossAxisAlignment: CrossAxisAlignment.center,
|
||||
children: [
|
||||
AnimatedBuilder(
|
||||
animation: _listeningAnimationController,
|
||||
builder: (context, _) {
|
||||
final t = _listeningAnimationController.value;
|
||||
final waveA =
|
||||
0.4 + 0.6 * (1 - ((t - 0.2).abs() * 2).clamp(0.0, 1.0));
|
||||
final waveB =
|
||||
0.4 + 0.6 * (1 - ((t - 0.5).abs() * 2).clamp(0.0, 1.0));
|
||||
final waveC =
|
||||
0.4 + 0.6 * (1 - ((t - 0.8).abs() * 2).clamp(0.0, 1.0));
|
||||
return Row(
|
||||
crossAxisAlignment: CrossAxisAlignment.center,
|
||||
children: [
|
||||
_buildWaveDot(scale: waveA),
|
||||
const SizedBox(width: 6),
|
||||
_buildWaveDot(scale: waveB),
|
||||
const SizedBox(width: 6),
|
||||
_buildWaveDot(scale: waveC),
|
||||
],
|
||||
);
|
||||
},
|
||||
),
|
||||
const SizedBox(width: 10),
|
||||
const Text(
|
||||
'正在聆听...',
|
||||
style: TextStyle(fontSize: 14, color: AppColors.slate500),
|
||||
),
|
||||
],
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
Widget _buildWaveDot({required double scale}) {
|
||||
return Transform.scale(
|
||||
scale: scale,
|
||||
child: Container(
|
||||
width: _recordingDotSize,
|
||||
height: _recordingDotSize,
|
||||
decoration: const BoxDecoration(
|
||||
shape: BoxShape.circle,
|
||||
color: AppColors.red600,
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
Future<void> _startRecording() async {
|
||||
try {
|
||||
await _voiceRecorder.start();
|
||||
_listeningAnimationController.repeat();
|
||||
if (!mounted) {
|
||||
return;
|
||||
}
|
||||
setState(() {
|
||||
_isRecording = true;
|
||||
});
|
||||
} catch (error) {
|
||||
if (!mounted) {
|
||||
return;
|
||||
}
|
||||
Toast.show(context, _readableError(error), type: ToastType.error);
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _stopRecording({bool autoSendAfterTranscribe = false}) async {
|
||||
String? audioPath;
|
||||
try {
|
||||
audioPath = await _voiceRecorder.stop();
|
||||
_listeningAnimationController.stop();
|
||||
if (!mounted) {
|
||||
return;
|
||||
}
|
||||
setState(() {
|
||||
_isRecording = false;
|
||||
_isTranscribing = true;
|
||||
});
|
||||
if (audioPath == null || audioPath.isEmpty) {
|
||||
throw StateError('录音失败，请重试');
|
||||
}
|
||||
final transcript = await _transcribeAudio(audioPath);
|
||||
if (!mounted) {
|
||||
return;
|
||||
}
|
||||
_messageController.text = transcript;
|
||||
_messageController.selection = TextSelection.fromPosition(
|
||||
TextPosition(offset: transcript.length),
|
||||
);
|
||||
if (autoSendAfterTranscribe && transcript.trim().isNotEmpty) {
|
||||
_messageController.clear();
|
||||
await _autoSendTranscript(transcript.trim());
|
||||
}
|
||||
} catch (error) {
|
||||
if (!mounted) {
|
||||
return;
|
||||
}
|
||||
Toast.show(context, _readableError(error), type: ToastType.error);
|
||||
} finally {
|
||||
if (audioPath != null) {
|
||||
final file = File(audioPath);
|
||||
if (await file.exists()) {
|
||||
await file.delete();
|
||||
}
|
||||
}
|
||||
if (mounted) {
|
||||
setState(() {
|
||||
_isTranscribing = false;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
String _readableError(Object error) {
|
||||
if (error is ApiException) {
|
||||
return error.message;
|
||||
}
|
||||
final raw = error.toString();
|
||||
if (raw.startsWith('Instance of')) {
|
||||
return '请求失败，请稍后重试';
|
||||
}
|
||||
return raw.replaceFirst('Bad state: ', '');
|
||||
}
|
||||
|
||||
void _showBottomSheet(BuildContext context) {
|
||||
showModalBottomSheet(
|
||||
context: context,
|
||||
|
||||
Reference in New Issue
Block a user