From b0fcb4b6304619bc301120971f56c7dcd3dd6328 Mon Sep 17 00:00:00 2001 From: Sikander Saleem Date: Tue, 27 Oct 2026 10:22:18 +0300 Subject: [PATCH] live session auto voice detection. --- .../copilot/controller/copilot_provider.dart | 80 +++++++++++++++---- .../screens/copilot_live_talk_page.dart | 28 +++++-- 2 files changed, 86 insertions(+), 22 deletions(-) diff --git a/lib/modules/copilot/controller/copilot_provider.dart b/lib/modules/copilot/controller/copilot_provider.dart index 23b9f14e..3814c2fc 100644 --- a/lib/modules/copilot/controller/copilot_provider.dart +++ b/lib/modules/copilot/controller/copilot_provider.dart @@ -54,18 +54,28 @@ class CopilotProvider extends ChangeNotifier { DateTime? _pttSpeechCandidateAt; DateTime? _pttLastAudioChunkAt; DateTime? _pttCalibrationStartedAt; + DateTime? _pttLastLoudAt; + DateTime? _pttLastLevelLogAt; + DateTime? _pttListeningStartedAt; Timer? _pttSilenceTimer; bool _pttAutoStopTriggered = false; bool _pttHasDetectedSpeech = false; bool _pttDetectorActive = false; bool _pttCalibrated = false; double _pttNoiseFloor = 0.0; - double _pttCalibrationSum = 0.0; - int _pttCalibrationSamples = 0; - static const Duration _silenceHoldDuration = Duration(seconds: 3); + final List _pttCalibrationLevels = []; + static const Duration _silenceHoldDuration = Duration(milliseconds: 1500); static const Duration _minimumSpeechDuration = Duration(milliseconds: 300); + // Short dips between syllables/words shorter than this don't reset speech detection. + static const Duration _speechGapTolerance = Duration(milliseconds: 250); + // Bounds for the measured background noise level (RMS, 0..1). + static const double _minNoiseFloor = 0.001; + static const double _maxNoiseFloor = 0.2; + // Safety net: send what was recorded if noise keeps the user "speaking" this long. + static const Duration _maxUtteranceDuration = Duration(seconds: 15); static const Duration _calibrationDuration = Duration(milliseconds: 700); static const Duration _audioChunkTimeout = Duration(seconds: 2); + static const Duration _noSpeechTimeout = Duration(seconds: 8); CopilotMessage? _pttCurrentAssistantMsg; final audioplayers.AudioPlayer _livePlayer = audioplayers.AudioPlayer(); final List _liveAudioQueue = []; @@ -719,6 +729,11 @@ class CopilotProvider extends ChangeNotifier { codec: sound.Codec.pcm16, numChannels: 1, sampleRate: 48000, + // OS-level noise suppression / echo cancellation / gain control before audio is sent for transcription. + audioSource: sound.AudioSource.voice_communication, // Android: voice-tuned mic with platform NS/AEC/AGC + enableNoiseSuppression: true, // Android: NoiseSuppressor effect on the PCM stream + enableEchoCancellation: true, // Android: removes the bot's own voice if picked up + enableVoiceProcessing: true, // iOS: Apple voice processing (NS/AEC/AGC) ); } catch (_) { _stopSilenceDetection(); @@ -740,9 +755,11 @@ class CopilotProvider extends ChangeNotifier { _pttSpeechCandidateAt = null; _pttLastAudioChunkAt = null; _pttNoiseFloor = 0; - _pttCalibrationSum = 0; - _pttCalibrationSamples = 0; + _pttCalibrationLevels.clear(); _pttCalibrationStartedAt = null; + _pttLastLoudAt = null; + _pttLastLevelLogAt = null; + _pttListeningStartedAt = DateTime.now(); _pttSilenceTimer = Timer.periodic( const Duration(milliseconds: 100), @@ -760,6 +777,9 @@ class CopilotProvider extends ChangeNotifier { _pttSpeechCandidateAt = null; _pttLastAudioChunkAt = null; _pttCalibrationStartedAt = null; + _pttLastLoudAt = null; + _pttLastLevelLogAt = null; + _pttListeningStartedAt = null; } void _processSilenceDetection(Uint8List chunk) { @@ -774,12 +794,15 @@ class CopilotProvider extends ChangeNotifier { if (!_pttCalibrated) { _pttCalibrationStartedAt ??= now; - _pttCalibrationSum += rms; - _pttCalibrationSamples++; + _pttCalibrationLevels.add(rms); if (now.difference(_pttCalibrationStartedAt!) < _calibrationDuration) { return; } - _pttNoiseFloor = (_pttCalibrationSum / _pttCalibrationSamples).clamp(0.001, 0.15).toDouble(); + // A quiet percentile (not the mean): steady background noise sets the baseline, + // while the user starting to talk during calibration doesn't inflate it. + final levels = [..._pttCalibrationLevels]..sort(); + _pttNoiseFloor = levels[(levels.length * 0.3).floor()].clamp(_minNoiseFloor, _maxNoiseFloor).toDouble(); + _pttLog('live: calibrated noiseFloor=${_pttNoiseFloor.toStringAsFixed(4)}'); _pttCalibrated = true; return; } @@ -787,38 +810,65 @@ class CopilotProvider extends ChangeNotifier { final speechThreshold = max(0.008, _pttNoiseFloor * 2.5); final isSpeechCandidate = rms > speechThreshold; + if (_pttLastLevelLogAt == null || now.difference(_pttLastLevelLogAt!) >= const Duration(seconds: 1)) { + _pttLastLevelLogAt = now; + _pttLog('live: rms=${rms.toStringAsFixed(4)} threshold=${speechThreshold.toStringAsFixed(4)} speechDetected=$_pttHasDetectedSpeech'); + } + if (isSpeechCandidate) { + _pttLastLoudAt = now; _pttSpeechCandidateAt ??= now; - if (now.difference(_pttSpeechCandidateAt!) >= _minimumSpeechDuration) { + if (_pttHasDetectedSpeech || now.difference(_pttSpeechCandidateAt!) >= _minimumSpeechDuration) { _pttHasDetectedSpeech = true; _pttLastSpeechAt = now; } return; } + // A brief dip between syllables is still part of the same utterance. + if (_pttLastLoudAt != null && now.difference(_pttLastLoudAt!) < _speechGapTolerance) return; + _pttSpeechCandidateAt = null; - if (!_pttHasDetectedSpeech) { - _pttNoiseFloor = (_pttNoiseFloor * 0.95 + rms * 0.05).clamp(0.001, 0.15).toDouble(); - } + // Follow the background on quiet chunks: drop fast if it got quieter, rise slowly if louder. + final rate = rms < _pttNoiseFloor ? 0.2 : 0.05; + _pttNoiseFloor = (_pttNoiseFloor * (1 - rate) + rms * rate).clamp(_minNoiseFloor, _maxNoiseFloor).toDouble(); } void _checkSpeechSilence() { - if (!_pttDetectorActive || _pttAutoStopTriggered || !_pttHasDetectedSpeech || pttStatus != PttStatus.recording || !liveSessionActive) { + if (!_pttDetectorActive || _pttAutoStopTriggered || pttStatus != PttStatus.recording || !liveSessionActive) { return; } final now = DateTime.now(); + if (!_pttHasDetectedSpeech) { + // User hasn't spoken since listening started: end the session instead of recording forever. + final startedAt = _pttListeningStartedAt; + if (startedAt != null && now.difference(startedAt) >= _noSpeechTimeout) { + _pttLog('live: no speech detected, stopping session'); + _pttAutoStopTriggered = true; + liveSessionActive = false; + _stopLivePlayback(); + cancelPushToTalk().then((_) { + pttErrorMessage = "Can't hear you, so the session was stopped. Tap the button to start again."; + notifyListeners(); + }).catchError((_) {}); + } + return; + } + final lastChunk = _pttLastAudioChunkAt; if (lastChunk == null || now.difference(lastChunk) > _audioChunkTimeout) return; final lastSpeech = _pttLastSpeechAt; - if (lastSpeech == null || now.difference(lastSpeech) < _silenceHoldDuration) return; + final startedAt = _pttListeningStartedAt; + final tooLong = startedAt != null && now.difference(startedAt) >= _maxUtteranceDuration; + if (!tooLong && (lastSpeech == null || now.difference(lastSpeech) < _silenceHoldDuration)) return; _pttAutoStopTriggered = true; _pttDetectorActive = false; _pttSilenceTimer?.cancel(); _pttSilenceTimer = null; - _pttLog('live: silence after speech, auto-stopping recording'); + _pttLog('live: ${tooLong ? 'max utterance length reached' : 'silence after speech'}, auto-stopping recording'); endPushToTalk().catchError((_) {}); } diff --git a/lib/modules/copilot/screens/copilot_live_talk_page.dart b/lib/modules/copilot/screens/copilot_live_talk_page.dart index 9b9eece3..6a6f0c73 100644 --- a/lib/modules/copilot/screens/copilot_live_talk_page.dart +++ b/lib/modules/copilot/screens/copilot_live_talk_page.dart @@ -1,10 +1,12 @@ import 'package:flutter/material.dart'; +import 'package:permission_handler/permission_handler.dart'; import 'package:provider/provider.dart'; import 'package:test_sa/border_beam/src/beam_style.dart'; import 'package:test_sa/border_beam/src/border_beam.dart'; import 'package:test_sa/controllers/providers/api/user_provider.dart'; import 'package:test_sa/extensions/context_extension.dart'; import 'package:test_sa/extensions/int_extensions.dart'; +import 'package:test_sa/extensions/string_extensions.dart'; import 'package:test_sa/extensions/text_extensions.dart'; import 'package:test_sa/extensions/widget_extensions.dart'; import 'package:test_sa/new_views/app_style/app_color.dart'; @@ -33,6 +35,18 @@ class _CopilotLiveTalkPageState extends State with WidgetsB @override void initState() { super.initState(); + makeConnection(); + } + + /// Microphone is only needed once the user starts the session, so ask on button tap. + Future _ensureMicPermission() async { + if (await Permission.microphone.isGranted) return true; + if ((await Permission.microphone.request()).isGranted) return true; + "Microphone permission denied. Please allow microphone access to use Live Session.".showToast; + return false; + } + + void makeConnection() { WidgetsBinding.instance.addObserver(this); WidgetsBinding.instance.addPostFrameCallback((_) { final provider = context.read(); @@ -89,14 +103,15 @@ class _CopilotLiveTalkPageState extends State with WidgetsB } } - void _onMainButtonTap(CopilotProvider provider) { - // The button only starts/stops the whole session (never per-utterance), and - // stopping must work at any point (listening / thinking / speaking). + /// The button only starts/stops the whole session. Within a session the provider + /// auto-sends when the user stops speaking, plays the answer, then listens again. + Future _onMainButtonTap(CopilotProvider provider) async { if (provider.liveSessionActive) { provider.stopLiveTalk().catchError((_) {}); - } else { - provider.startLiveTalk(context.settingProvider.language!).catchError((_) {}); + return; } + if (!await _ensureMicPermission() || !mounted) return; + provider.startLiveTalk(context.settingProvider.language!).catchError((_) {}); } Widget _buildConnectionPlaceholder(BuildContext context, CopilotProvider provider) { @@ -128,7 +143,7 @@ class _CopilotLiveTalkPageState extends State with WidgetsB @override Widget build(BuildContext context) { return Scaffold( - appBar: const DefaultAppBar(title: "Live Session"), + appBar: const DefaultAppBar(title: "Live Session", showCopilot: false), body: Column( children: [ Expanded( @@ -254,4 +269,3 @@ class _CopilotLiveTalkPageState extends State with WidgetsB ); } } -