diff --git a/lib/modules/copilot/controller/copilot_provider.dart b/lib/modules/copilot/controller/copilot_provider.dart index c83c8e5c..5d82fafa 100644 --- a/lib/modules/copilot/controller/copilot_provider.dart +++ b/lib/modules/copilot/controller/copilot_provider.dart @@ -61,9 +61,15 @@ class CopilotProvider extends ChangeNotifier { DateTime? _pttLastLoudAt; bool _pttAutoStopTriggered = false; bool _pttHasDetectedSpeech = false; - // Adaptive noise floor (RMS) learned from quiet chunks so detection works - // regardless of device mic gain / ambient noise. + // Self-calibrating VAD: we continuously track the quiet "floor" AND the loud + // speech "peak", then put the speaking threshold in the gap between them, so + // detection adapts to any device mic gain / ambient noise level. double _pttNoiseFloor = 0.01; + double _pttSpeechPeak = 0.05; + // Smoothed voice-activity (0..1 moving average of "is this chunk speech?"). + // Brief room-noise/echo spikes can't reset the silence timer — only sustained + // speech keeps this high, which is what we key the auto-stop off of. + double _pttVoiceActivity = 0.0; static const Duration _silenceHoldDuration = Duration(milliseconds: 3000); CopilotMessage? _pttCurrentAssistantMsg; @@ -392,6 +398,38 @@ class CopilotProvider extends ChangeNotifier { await _stopHubAndCleanup(); _setHubState(PttHubState.connecting); + // signalr_netcore can throw a "Null check operator used on a null value" + // from HubConnection._startInternal when the server closes the long-poll + // connection mid-handshake (a package race on _handshakeCompleter). It is + // transient, so retry a few times before surfacing an error. + const maxAttempts = 3; + Object? lastError; + for (var attempt = 1; attempt <= maxAttempts; attempt++) { + try { + await _buildAndStartHub(); + return; // connected + } catch (e, st) { + lastError = e; + _pttLog('connectPushToTalkHub: attempt $attempt/$maxAttempts failed error=$e'); + await _stopHubAndCleanup(); + if (attempt < maxAttempts) { + await Future.delayed(Duration(milliseconds: 400 * attempt)); + } else { + _pttLog('connectPushToTalkHub: giving up\n$st'); + } + } + } + + pttErrorMessage = "Failed to connect to push-to-talk service"; + pttStatus = PttStatus.error; + pttHubState = PttHubState.disconnected; + notifyListeners(); + throw lastError ?? StateError('Failed to connect'); + } + + /// Builds a single fresh hub and starts it. Throws on failure so the caller + /// ([_doConnectPushToTalkHub]) can retry. + Future _buildAndStartHub() async { final token = ApiManager.instance.user?.token ?? ''; final url = "${URLs.copilotHubUrl}?access_token=$token"; _pttLog('connectPushToTalkHub: building connection url=${URLs.copilotHubUrl}'); @@ -442,33 +480,23 @@ class CopilotProvider extends ChangeNotifier { _handleConnectionInterrupted("Connection lost. Tap retry to reconnect."); }); - _pttHub!.on("RecordingStarted", _onRecordingStarted); - _pttHub!.on("RecordingStopped", _onRecordingStopped); - _pttHub!.on("RecordingCancelled", _onRecordingCancelled); - _pttHub!.on("TranscriptionReceived", _onTranscriptionReceived); - _pttHub!.on("ResponseReceived", _onResponseReceived); - _pttHub!.on("AudioReceived", _onAudioReceived); - _pttHub!.on("RecordingError", _onRecordingError); - - try { - _pttLog('connectPushToTalkHub: calling start()...'); - await _pttHub!.start()?.timeout(const Duration(seconds: 20), onTimeout: () { - throw TimeoutException('Connection to push-to-talk service timed out'); - }); - _pttLog('connectPushToTalkHub: connected, state=${_pttHub!.state}'); - pttHubState = PttHubState.connected; - if (pttStatus == PttStatus.error) pttStatus = PttStatus.idle; - pttErrorMessage = null; - notifyListeners(); - } catch (e, st) { - _pttLog('connectPushToTalkHub: failed error=$e\n$st'); - pttErrorMessage = "Failed to connect to push-to-talk service"; - pttStatus = PttStatus.error; - await _stopHubAndCleanup(); - pttHubState = PttHubState.disconnected; - notifyListeners(); - rethrow; - } + hub.on("RecordingStarted", _onRecordingStarted); + hub.on("RecordingStopped", _onRecordingStopped); + hub.on("RecordingCancelled", _onRecordingCancelled); + hub.on("TranscriptionReceived", _onTranscriptionReceived); + hub.on("ResponseReceived", _onResponseReceived); + hub.on("AudioReceived", _onAudioReceived); + hub.on("RecordingError", _onRecordingError); + + _pttLog('connectPushToTalkHub: calling start()...'); + await hub.start()?.timeout(const Duration(seconds: 20), onTimeout: () { + throw TimeoutException('Connection to push-to-talk service timed out'); + }); + _pttLog('connectPushToTalkHub: connected, state=${hub.state}'); + pttHubState = PttHubState.connected; + if (pttStatus == PttStatus.error) pttStatus = PttStatus.idle; + pttErrorMessage = null; + notifyListeners(); } void _setHubState(PttHubState state) { @@ -623,15 +651,17 @@ class CopilotProvider extends ChangeNotifier { await beginPushToTalk(language, isLive: true); } - /// Stops the current Live Talk session (does not auto-restart listening again). + /// Stops the current Live Talk session completely. The button is only used to + /// start/stop the whole session, so ending it must hard-stop everything: + /// no more listening, thinking, or speaking until the user taps to start again. Future stopLiveTalk() async { + _pttLog('stopLiveTalk: ending session, stopping everything'); liveSessionActive = false; + _pttIsLive = false; + _pttAutoStopTriggered = true; // block any in-flight auto-stop + _cancelResponseWatchdog(); _stopLivePlayback(); - if (pttStatus == PttStatus.recording) { - await endPushToTalk(); - } else { - await cancelPushToTalk(); - } + await cancelPushToTalk(); } Future endPushToTalk() async { @@ -731,33 +761,60 @@ class CopilotProvider extends ChangeNotifier { _pttHasDetectedSpeech = false; _pttLastLoudAt = null; _pttNoiseFloor = 0.01; + _pttSpeechPeak = 0.05; + _pttVoiceActivity = 0.0; } void _stopSilenceDetection() { _pttLastLoudAt = null; _pttHasDetectedSpeech = false; _pttAutoStopTriggered = false; + _pttVoiceActivity = 0.0; } /// Live Talk auto-stop: once the user has spoken and then stayed silent for /// [_silenceHoldDuration] (3s), automatically send StopRecording — the button /// is only used to start/stop the whole session, never per-utterance. - /// Uses an adaptive noise floor so it works across devices/mic gains. + /// + /// Self-calibrating + noise/echo tolerant: + /// - Tracks the quiet floor (fast down / slow up) and the speech peak + /// (fast up / slow down), and sets a per-chunk threshold 45% of the way + /// from floor toward peak. + /// - Smooths that per-chunk decision into [_pttVoiceActivity] so a few + /// stray room-noise / echo chunks can't masquerade as speech and keep + /// resetting the silence countdown. Only *sustained* speech counts. void _processSilenceDetection(Uint8List chunk) { if (_pttAutoStopTriggered) return; final rms = _chunkRms(chunk); final now = DateTime.now(); - // Speaking if clearly above the learned noise floor (with an absolute min). - final speechThreshold = (_pttNoiseFloor * 2.2).clamp(0.008, 0.3); - final speaking = rms > speechThreshold; - _pttLog('silence: rms=${rms.toStringAsFixed(4)} floor=${_pttNoiseFloor.toStringAsFixed(4)} thr=${speechThreshold.toStringAsFixed(4)} speaking=$speaking spoke=$_pttHasDetectedSpeech'); - if (speaking) { + + if (rms < _pttNoiseFloor) { + _pttNoiseFloor = rms; + } else { + _pttNoiseFloor = _pttNoiseFloor * 0.995 + rms * 0.005; + } + if (rms > _pttSpeechPeak) { + _pttSpeechPeak = rms; + } else { + _pttSpeechPeak = _pttSpeechPeak * 0.995 + rms * 0.005; + } + + final span = _pttSpeechPeak - _pttNoiseFloor; + final threshold = max(_pttNoiseFloor + span * 0.45, 0.012); + final chunkIsSpeech = rms > threshold && span > 0.01; + + // Smooth: a single loud chunk only nudges activity, sustained speech climbs. + _pttVoiceActivity = _pttVoiceActivity * 0.8 + (chunkIsSpeech ? 1.0 : 0.0) * 0.2; + // Hysteresis: need clear sustained energy to count as actively speaking. + final activelySpeaking = _pttVoiceActivity > 0.4; + + _pttLog('silence: rms=${rms.toStringAsFixed(4)} floor=${_pttNoiseFloor.toStringAsFixed(4)} peak=${_pttSpeechPeak.toStringAsFixed(4)} thr=${threshold.toStringAsFixed(4)} act=${_pttVoiceActivity.toStringAsFixed(2)} speaking=$activelySpeaking spoke=$_pttHasDetectedSpeech'); + + if (activelySpeaking) { _pttLastLoudAt = now; _pttHasDetectedSpeech = true; return; } - // Quiet chunk: slowly adapt the noise floor toward the current level. - _pttNoiseFloor = _pttNoiseFloor * 0.9 + rms * 0.1; if (!_pttHasDetectedSpeech) return; final lastLoud = _pttLastLoudAt; if (lastLoud != null && now.difference(lastLoud) >= _silenceHoldDuration) { diff --git a/lib/modules/copilot/screens/copilot_live_talk_page.dart b/lib/modules/copilot/screens/copilot_live_talk_page.dart index a0dbfe31..8d499d39 100644 --- a/lib/modules/copilot/screens/copilot_live_talk_page.dart +++ b/lib/modules/copilot/screens/copilot_live_talk_page.dart @@ -13,10 +13,6 @@ import 'package:test_sa/thinking_orbs/src/thinking_orb.dart'; import '../controller/copilot_provider.dart'; import '../widgets/copilot_message_bubble.dart'; -/// Live Talk realtime voice screen for Copilot. -/// Unlike Push-to-Talk (press & hold), this is a continuous conversation mode: -/// tap once to start, the app keeps listening/responding automatically until -/// the user taps again to stop. class CopilotLiveTalkPage extends StatefulWidget { const CopilotLiveTalkPage({Key? key}) : super(key: key); @@ -79,7 +75,7 @@ class _CopilotLiveTalkPageState extends State with WidgetsB String _statusLabel(PttStatus status, bool isLiveActive) { switch (status) { case PttStatus.idle: - return isLiveActive ? "Getting ready…" : "Tap to start Live Talk"; + return isLiveActive ? "Getting ready…" : "Tap to start Live Session"; case PttStatus.connecting: return "Connecting…"; case PttStatus.recording: @@ -135,7 +131,7 @@ class _CopilotLiveTalkPageState extends State with WidgetsB @override Widget build(BuildContext context) { return Scaffold( - appBar: const DefaultAppBar(title: "Live Talk"), + appBar: const DefaultAppBar(title: "Live Session"), body: Column( children: [ Expanded( @@ -248,7 +244,7 @@ class _CopilotLiveTalkPageState extends State with WidgetsB 8.height, if (isActive) Text( - "Tap to stop", + "Tap to stop session", style: AppTextStyles.bodyText2.copyWith(color: AppColor.lightTextColor(context)), ), 24.height,