diff --git a/lib/modules/copilot/controller/copilot_provider.dart b/lib/modules/copilot/controller/copilot_provider.dart index 91e7182e..c83c8e5c 100644 --- a/lib/modules/copilot/controller/copilot_provider.dart +++ b/lib/modules/copilot/controller/copilot_provider.dart @@ -3,8 +3,10 @@ import 'dart:convert'; import 'dart:developer' as log1; import 'dart:io'; import 'dart:math'; +import 'dart:typed_data'; import 'package:signalr_netcore/ihub_protocol.dart'; import 'package:signalr_netcore/signalr_client.dart'; +import 'package:audioplayers/audioplayers.dart' as audioplayers; import 'package:flutter/foundation.dart'; import 'package:flutter_sound/flutter_sound.dart' as sound; import 'package:http/http.dart'; @@ -54,17 +56,30 @@ class CopilotProvider extends ChangeNotifier { Future _pttSendQueue = Future.value(); int _pttChunkCount = 0; - // Live Talk voice-activity (silence) detection. - StreamSubscription? _pttProgressSub; + // Live Talk voice-activity (silence) detection, computed from the PCM mic + // chunks (onProgress metering is not emitted when recording to a stream). DateTime? _pttLastLoudAt; - DateTime? _pttRecordingStartedAt; bool _pttAutoStopTriggered = false; bool _pttHasDetectedSpeech = false; - static const double _silenceDbThreshold = 38.0; - static const Duration _silenceHoldDuration = Duration(milliseconds: 1400); + // Adaptive noise floor (RMS) learned from quiet chunks so detection works + // regardless of device mic gain / ambient noise. + double _pttNoiseFloor = 0.01; + static const Duration _silenceHoldDuration = Duration(milliseconds: 3000); CopilotMessage? _pttCurrentAssistantMsg; + // Live Talk: the provider plays the bot's TTS audio itself so it can resume + // listening only AFTER the bot has finished speaking (avoids recording the + // bot's own voice). Push-to-Talk does not use any of this. + final audioplayers.AudioPlayer _livePlayer = audioplayers.AudioPlayer(); + final List _liveAudioQueue = []; + bool _liveIsPlaying = false; + Timer? _liveRestartTimer; + + CopilotProvider() { + _livePlayer.onPlayerComplete.listen((_) => _onLiveClipComplete()); + } + void _pttLog(String message) => log1.log('[PTT] $message'); Future loadHistory() async { @@ -611,6 +626,7 @@ class CopilotProvider extends ChangeNotifier { /// Stops the current Live Talk session (does not auto-restart listening again). Future stopLiveTalk() async { liveSessionActive = false; + _stopLivePlayback(); if (pttStatus == PttStatus.recording) { await endPushToTalk(); } else { @@ -689,6 +705,7 @@ class CopilotProvider extends ChangeNotifier { _pttChunkCount++; _pttLog('mic chunk #$_pttChunkCount captured bytes=${chunk.length}'); _enqueueChunkSend(chunk); + if (_pttIsLive) _processSilenceDetection(chunk); }); await _pttRecorder.startRecorder( @@ -707,41 +724,60 @@ class CopilotProvider extends ChangeNotifier { } } - /// Monitors mic decibel levels while in Live Talk mode and automatically - /// triggers [endPushToTalk] once the user has been silent for - /// [_silenceHoldDuration] AFTER having spoken, so the user doesn't need to - /// tap anything. Won't trigger on the initial silence before the user - /// starts speaking. + /// Resets voice-activity state for a new Live Talk recording. Actual silence + /// detection happens per mic chunk in [_processSilenceDetection]. void _startSilenceDetection() { _pttAutoStopTriggered = false; _pttHasDetectedSpeech = false; - _pttRecordingStartedAt = DateTime.now(); _pttLastLoudAt = null; - _pttProgressSub?.cancel(); - _pttProgressSub = _pttRecorder.onProgress?.listen((disposition) { - final db = disposition.decibels ?? 0; - final now = DateTime.now(); - if (db > _silenceDbThreshold) { - _pttLastLoudAt = now; - _pttHasDetectedSpeech = true; - return; - } - if (_pttAutoStopTriggered || !_pttHasDetectedSpeech) return; - final lastLoud = _pttLastLoudAt; - if (lastLoud != null && now.difference(lastLoud) >= _silenceHoldDuration) { - _pttAutoStopTriggered = true; - _pttLog('_startSilenceDetection: silence detected after speech, auto-stopping recording (live talk)'); - endPushToTalk().catchError((_) {}); - } - }); + _pttNoiseFloor = 0.01; } void _stopSilenceDetection() { - _pttProgressSub?.cancel(); - _pttProgressSub = null; _pttLastLoudAt = null; - _pttRecordingStartedAt = null; _pttHasDetectedSpeech = false; + _pttAutoStopTriggered = false; + } + + /// Live Talk auto-stop: once the user has spoken and then stayed silent for + /// [_silenceHoldDuration] (3s), automatically send StopRecording — the button + /// is only used to start/stop the whole session, never per-utterance. + /// Uses an adaptive noise floor so it works across devices/mic gains. + void _processSilenceDetection(Uint8List chunk) { + if (_pttAutoStopTriggered) return; + final rms = _chunkRms(chunk); + final now = DateTime.now(); + // Speaking if clearly above the learned noise floor (with an absolute min). + final speechThreshold = (_pttNoiseFloor * 2.2).clamp(0.008, 0.3); + final speaking = rms > speechThreshold; + _pttLog('silence: rms=${rms.toStringAsFixed(4)} floor=${_pttNoiseFloor.toStringAsFixed(4)} thr=${speechThreshold.toStringAsFixed(4)} speaking=$speaking spoke=$_pttHasDetectedSpeech'); + if (speaking) { + _pttLastLoudAt = now; + _pttHasDetectedSpeech = true; + return; + } + // Quiet chunk: slowly adapt the noise floor toward the current level. + _pttNoiseFloor = _pttNoiseFloor * 0.9 + rms * 0.1; + if (!_pttHasDetectedSpeech) return; + final lastLoud = _pttLastLoudAt; + if (lastLoud != null && now.difference(lastLoud) >= _silenceHoldDuration) { + _pttAutoStopTriggered = true; + _pttLog('live: silence after speech, auto-stopping recording'); + endPushToTalk().catchError((_) {}); + } + } + + /// Root-mean-square amplitude (0..1) of a PCM16 mono little-endian chunk. + double _chunkRms(Uint8List chunk) { + final sampleCount = chunk.length ~/ 2; + if (sampleCount == 0) return 0; + final data = ByteData.sublistView(chunk); + double sumSquares = 0; + for (var i = 0; i < sampleCount; i++) { + final sample = data.getInt16(i * 2, Endian.little) / 32768.0; + sumSquares += sample * sample; + } + return sqrt(sumSquares / sampleCount); } Future _stopMicStreamingAndFlush() async { @@ -839,13 +875,17 @@ class CopilotProvider extends ChangeNotifier { final text = data?['voiceAnswer']?.toString() ?? ''; _pttCurrentAssistantMsg = CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.assistant, content: text, timestamp: DateTime.now()); pttMessages.add(_pttCurrentAssistantMsg!); - pttStatus = PttStatus.idle; - notifyListeners(); if (liveSessionActive && _pttIsLive) { - final lang = _pttLanguage ?? '1'; - _pttLog('ResponseReceived: live session active, auto-restarting listening'); - Future.microtask(() => beginPushToTalk(lang, isLive: true).catchError((_) {})); + // Live Talk: let the bot finish speaking its TTS audio, then resume + // listening. _maybeRestartLiveListening is debounced so it waits for any + // audio clips that arrive right after this text response. + pttStatus = PttStatus.playingAudio; + notifyListeners(); + _maybeRestartLiveListening(); + } else { + pttStatus = PttStatus.idle; + notifyListeners(); } } @@ -854,13 +894,69 @@ class CopilotProvider extends ChangeNotifier { try { final bytes = base64Decode(params.first.toString()); _pttLog('event AudioReceived bytes=${bytes.length}'); - _pttCurrentAssistantMsg?.ttsAudioClips.add(bytes); - notifyListeners(); + if (liveSessionActive && _pttIsLive) { + // Live Talk: queue and play through the provider so we know when the + // bot finished speaking (to resume listening). No widget player. + _liveAudioQueue.add(bytes); + _playNextLiveClip(); + } else { + _pttCurrentAssistantMsg?.ttsAudioClips.add(bytes); + notifyListeners(); + } } catch (e) { _pttLog('_onAudioReceived: failed to decode error=$e'); } } + // ----- Live Talk audio playback + auto-resume listening ----- + + void _playNextLiveClip() { + if (_liveIsPlaying || _liveAudioQueue.isEmpty) return; + _liveRestartTimer?.cancel(); + final clip = _liveAudioQueue.removeAt(0); + _liveIsPlaying = true; + if (pttStatus != PttStatus.playingAudio) { + pttStatus = PttStatus.playingAudio; + notifyListeners(); + } + _livePlayer.play(audioplayers.BytesSource(clip, mimeType: 'audio/wav')).catchError((e) { + _pttLog('_playNextLiveClip: play error=$e'); + _onLiveClipComplete(); + }); + } + + void _onLiveClipComplete() { + _liveIsPlaying = false; + if (_liveAudioQueue.isNotEmpty) { + _playNextLiveClip(); + } else { + _maybeRestartLiveListening(); + } + } + + /// After the bot stops speaking, resume listening. Debounced so trailing + /// audio clips (arriving just after) don't cause a premature restart. + void _maybeRestartLiveListening() { + if (!liveSessionActive || !_pttIsLive) return; + _liveRestartTimer?.cancel(); + _liveRestartTimer = Timer(const Duration(milliseconds: 600), () { + if (!liveSessionActive || !_pttIsLive) return; + if (_liveIsPlaying || _liveAudioQueue.isNotEmpty) return; + if (pttStatus == PttStatus.recording || pttStatus == PttStatus.connecting) return; + _pttLog('live: bot finished speaking, resuming listening'); + pttStatus = PttStatus.idle; + beginPushToTalk(_pttLanguage ?? '1', isLive: true).catchError((_) {}); + }); + } + + void _stopLivePlayback() { + _liveRestartTimer?.cancel(); + _liveRestartTimer = null; + _liveAudioQueue.clear(); + _liveIsPlaying = false; + _livePlayer.stop().catchError((_) {}); + } + void _onRecordingError(List? params) { final msg = (params != null && params.isNotEmpty) ? params.first?.toString() : null; _pttLog('event RecordingError message=$msg'); @@ -868,6 +964,7 @@ class CopilotProvider extends ChangeNotifier { pttErrorMessage = msg ?? "Recording error occurred"; pttStatus = PttStatus.error; liveSessionActive = false; + _stopLivePlayback(); _abortMicStreaming(); notifyListeners(); } @@ -887,6 +984,8 @@ class CopilotProvider extends ChangeNotifier { void dispose() { _streamSub?.cancel(); _cancelResponseWatchdog(); + _liveRestartTimer?.cancel(); + _livePlayer.dispose(); _abortMicStreaming(); if (_pttRecorderOpen) { _pttRecorder.closeRecorder(); diff --git a/lib/modules/copilot/screens/copilot_live_talk_page.dart b/lib/modules/copilot/screens/copilot_live_talk_page.dart index 5d7538b8..a0dbfe31 100644 --- a/lib/modules/copilot/screens/copilot_live_talk_page.dart +++ b/lib/modules/copilot/screens/copilot_live_talk_page.dart @@ -1,5 +1,7 @@ import 'package:flutter/material.dart'; import 'package:provider/provider.dart'; +import 'package:test_sa/border_beam/src/beam_style.dart'; +import 'package:test_sa/border_beam/src/border_beam.dart'; import 'package:test_sa/controllers/providers/api/user_provider.dart'; import 'package:test_sa/extensions/context_extension.dart'; import 'package:test_sa/extensions/int_extensions.dart'; @@ -7,6 +9,7 @@ import 'package:test_sa/extensions/text_extensions.dart'; import 'package:test_sa/extensions/widget_extensions.dart'; import 'package:test_sa/new_views/app_style/app_color.dart'; import 'package:test_sa/new_views/common_widgets/default_app_bar.dart'; +import 'package:test_sa/thinking_orbs/src/thinking_orb.dart'; import '../controller/copilot_provider.dart'; import '../widgets/copilot_message_bubble.dart'; @@ -21,7 +24,7 @@ class CopilotLiveTalkPage extends StatefulWidget { State createState() => _CopilotLiveTalkPageState(); } -class _CopilotLiveTalkPageState extends State { +class _CopilotLiveTalkPageState extends State with WidgetsBindingObserver { final ScrollController _scrollController = ScrollController(); CopilotProvider? _provider; @@ -34,6 +37,7 @@ class _CopilotLiveTalkPageState extends State { @override void initState() { super.initState(); + WidgetsBinding.instance.addObserver(this); WidgetsBinding.instance.addPostFrameCallback((_) { final provider = context.read(); provider.startNewPttConversation(); @@ -41,8 +45,19 @@ class _CopilotLiveTalkPageState extends State { }); } + @override + void didChangeAppLifecycleState(AppLifecycleState state) { + if (state == AppLifecycleState.paused) { + // OS suspends sockets/mic in background; stop the live session. + _provider?.stopLiveTalk().catchError((_) {}); + } else if (state == AppLifecycleState.resumed) { + _provider?.ensurePushToTalkConnected().catchError((_) {}); + } + } + @override void dispose() { + WidgetsBinding.instance.removeObserver(this); _scrollController.dispose(); _provider?.stopLiveTalk().catchError((_) {}); _provider?.disconnectPushToTalkHub().catchError((_) {}); @@ -79,10 +94,8 @@ class _CopilotLiveTalkPageState extends State { } void _onMainButtonTap(CopilotProvider provider) { - final status = provider.pttStatus; - final isBusy = status == PttStatus.connecting || status == PttStatus.thinking; - if (isBusy) return; - + // The button only starts/stops the whole session (never per-utterance), and + // stopping must work at any point (listening / thinking / speaking). if (provider.liveSessionActive) { provider.stopLiveTalk().catchError((_) {}); } else { @@ -90,6 +103,35 @@ class _CopilotLiveTalkPageState extends State { } } + /// Shown in place of the mic while the socket is connecting / disconnected. + Widget _buildConnectionPlaceholder(BuildContext context, CopilotProvider provider) { + // Treat "disconnected without an error" as connecting (e.g. the first frame + // before the connection attempt kicks off). + final isConnecting = provider.pttHubState == PttHubState.connecting || provider.pttErrorMessage == null; + return SizedBox( + height: 148, + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + mainAxisSize: MainAxisSize.min, + children: [ + if (isConnecting) ...[ + const SizedBox(height: 28, width: 28, child: CircularProgressIndicator(strokeWidth: 2.5)), + 12.height, + Text( + "Connecting…", + style: AppTextStyles.bodyText2.copyWith(color: AppColor.lightTextColor(context)), + ), + ] else + TextButton.icon( + onPressed: () => provider.ensurePushToTalkConnected().catchError((_) {}), + icon: const Icon(Icons.refresh_rounded), + label: const Text("Retry connection"), + ), + ], + ), + ).paddingOnly(top: 8, bottom: 8); + } + @override Widget build(BuildContext context) { return Scaffold( @@ -149,6 +191,10 @@ class _CopilotLiveTalkPageState extends State { SafeArea( child: Consumer( builder: (context, provider, _) { + // Hide the mic until the socket connection is established. + if (provider.pttHubState != PttHubState.connected) { + return _buildConnectionPlaceholder(context, provider); + } final status = provider.pttStatus; final isActive = provider.liveSessionActive; final isRecording = status == PttStatus.recording; @@ -162,30 +208,41 @@ class _CopilotLiveTalkPageState extends State { style: AppTextStyles.bodyText2.copyWith(color: AppColor.lightTextColor(context)), ), 16.height, - GestureDetector( - onTap: isBusy ? null : () => _onMainButtonTap(provider), - child: AnimatedContainer( - duration: const Duration(milliseconds: 150), - height: isRecording || isActive ? 96 : 80, - width: isRecording || isActive ? 96 : 80, - decoration: BoxDecoration( - shape: BoxShape.circle, - color: isRecording - ? Colors.red - : isActive - ? AppColor.primary10.withValues(alpha: 0.8) - : AppColor.primary10, + Container( + height: 100, + alignment: Alignment.center, + child: Container( + padding: const EdgeInsets.all(0), + decoration: BoxDecoration(color: AppColor.background(context), shape: BoxShape.circle, border: Border.all(color: Colors.transparent)), + child: BorderBeam( + size: BeamSize.pulseOutside, + colorVariant: BeamColorVariant.colorful, + reserveGlowSpace: false, + borderRadius: 50, + strength: 0.7, + child: GestureDetector( + onTap: () => _onMainButtonTap(provider), + child: AnimatedContainer( + duration: const Duration(milliseconds: 150), + height: isRecording || isActive ? 96 : 80, + width: isRecording || isActive ? 96 : 80, + child: isBusy + ? Padding( + padding: const EdgeInsets.all(12), + child: ThinkingOrb( + size: 48, + state: ThinkingOrbState.working, + theme: context.isDark ? ThinkingOrbTheme.dark : ThinkingOrbTheme.light, + ), + ) + : Icon( + isActive ? Icons.stop_rounded : Icons.graphic_eq_rounded, + color: AppColor.icon2Color(context), + size: isRecording || isActive ? 44 : 36, + ), + ), + ), ), - child: isBusy - ? const Padding( - padding: EdgeInsets.all(24), - child: CircularProgressIndicator(color: Colors.white, strokeWidth: 2.5), - ) - : Icon( - isActive ? Icons.stop_rounded : Icons.graphic_eq_rounded, - color: Colors.white, - size: isRecording || isActive ? 44 : 36, - ), ), ), 8.height,