Haroon6138
/
cloudsolutions-atoms
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 28 Wiki Activity

live session auto voice detection.

Browse Source
main_production_copilot
Sikander Saleem 1 day ago
parent c65439244b
commit b0fcb4b630
2 changed files with 86 additions and 22 deletions
Show Stats Download Patch File Download Diff File

80
lib/modules/copilot/controller/copilot_provider.dart
Unescape Escape View File

@ -54,18 +54,28 @@ class CopilotProvider extends ChangeNotifier {
DateTime? _pttSpeechCandidateAt;
DateTime? _pttLastAudioChunkAt;
DateTime? _pttCalibrationStartedAt;
DateTime? _pttLastLoudAt;
DateTime? _pttLastLevelLogAt;
DateTime? _pttListeningStartedAt;
Timer? _pttSilenceTimer;
bool _pttAutoStopTriggered = false;
bool _pttHasDetectedSpeech = false;
bool _pttDetectorActive = false;
bool _pttCalibrated = false;
double _pttNoiseFloor = 0.0;
double _pttCalibrationSum = 0.0;
int _pttCalibrationSamples = 0;
static const Duration _silenceHoldDuration = Duration(seconds: 3);
final List<double> _pttCalibrationLevels = [];
static const Duration _silenceHoldDuration = Duration(milliseconds: 1500);
static const Duration _minimumSpeechDuration = Duration(milliseconds: 300);
// Short dips between syllables/words shorter than this don't reset speech detection.
static const Duration _speechGapTolerance = Duration(milliseconds: 250);
// Bounds for the measured background noise level (RMS, 0..1).
static const double _minNoiseFloor = 0.001;
static const double _maxNoiseFloor = 0.2;
// Safety net: send what was recorded if noise keeps the user "speaking" this long.
static const Duration _maxUtteranceDuration = Duration(seconds: 15);
static const Duration _calibrationDuration = Duration(milliseconds: 700);
static const Duration _audioChunkTimeout = Duration(seconds: 2);
static const Duration _noSpeechTimeout = Duration(seconds: 8);
CopilotMessage? _pttCurrentAssistantMsg;
final audioplayers.AudioPlayer _livePlayer = audioplayers.AudioPlayer();
final List<Uint8List> _liveAudioQueue = [];
@ -719,6 +729,11 @@ class CopilotProvider extends ChangeNotifier {
codec: sound.Codec.pcm16,
numChannels: 1,
sampleRate: 48000,
// OS-level noise suppression / echo cancellation / gain control before audio is sent for transcription.
audioSource: sound.AudioSource.voice_communication, // Android: voice-tuned mic with platform NS/AEC/AGC
enableNoiseSuppression: true, // Android: NoiseSuppressor effect on the PCM stream
enableEchoCancellation: true, // Android: removes the bot's own voice if picked up
enableVoiceProcessing: true, // iOS: Apple voice processing (NS/AEC/AGC)
);
} catch (_) {
_stopSilenceDetection();
@ -740,9 +755,11 @@ class CopilotProvider extends ChangeNotifier {
_pttSpeechCandidateAt = null;
_pttLastAudioChunkAt = null;
_pttNoiseFloor = 0;
_pttCalibrationSum = 0;
_pttCalibrationSamples = 0;
_pttCalibrationLevels.clear();
_pttCalibrationStartedAt = null;
_pttLastLoudAt = null;
_pttLastLevelLogAt = null;
_pttListeningStartedAt = DateTime.now();
_pttSilenceTimer = Timer.periodic(
const Duration(milliseconds: 100),
@ -760,6 +777,9 @@ class CopilotProvider extends ChangeNotifier {
_pttSpeechCandidateAt = null;
_pttLastAudioChunkAt = null;
_pttCalibrationStartedAt = null;
_pttLastLoudAt = null;
_pttLastLevelLogAt = null;
_pttListeningStartedAt = null;
}
void _processSilenceDetection(Uint8List chunk) {
@ -774,12 +794,15 @@ class CopilotProvider extends ChangeNotifier {
if (!_pttCalibrated) {
_pttCalibrationStartedAt ??= now;
_pttCalibrationSum += rms;
_pttCalibrationSamples++;
_pttCalibrationLevels.add(rms);
if (now.difference(_pttCalibrationStartedAt!) < _calibrationDuration) {
return;
}
_pttNoiseFloor = (_pttCalibrationSum / _pttCalibrationSamples).clamp(0.001, 0.15).toDouble();
// A quiet percentile (not the mean): steady background noise sets the baseline,
// while the user starting to talk during calibration doesn't inflate it.
final levels = [..._pttCalibrationLevels]..sort();
_pttNoiseFloor = levels[(levels.length * 0.3).floor()].clamp(_minNoiseFloor, _maxNoiseFloor).toDouble();
_pttLog('live: calibrated noiseFloor=${_pttNoiseFloor.toStringAsFixed(4)}');
_pttCalibrated = true;
return;
}
@ -787,38 +810,65 @@ class CopilotProvider extends ChangeNotifier {
final speechThreshold = max(0.008, _pttNoiseFloor * 2.5);
final isSpeechCandidate = rms > speechThreshold;
if (_pttLastLevelLogAt == null || now.difference(_pttLastLevelLogAt!) >= const Duration(seconds: 1)) {
_pttLastLevelLogAt = now;
_pttLog('live: rms=${rms.toStringAsFixed(4)} threshold=${speechThreshold.toStringAsFixed(4)} speechDetected=$_pttHasDetectedSpeech');
}
if (isSpeechCandidate) {
_pttLastLoudAt = now;
_pttSpeechCandidateAt ??= now;
if (now.difference(_pttSpeechCandidateAt!) >= _minimumSpeechDuration) {
if (_pttHasDetectedSpeech || now.difference(_pttSpeechCandidateAt!) >= _minimumSpeechDuration) {
_pttHasDetectedSpeech = true;
_pttLastSpeechAt = now;
}
return;
}
// A brief dip between syllables is still part of the same utterance.
if (_pttLastLoudAt != null && now.difference(_pttLastLoudAt!) < _speechGapTolerance) return;
_pttSpeechCandidateAt = null;
if (!_pttHasDetectedSpeech) {
_pttNoiseFloor = (_pttNoiseFloor * 0.95 + rms * 0.05).clamp(0.001, 0.15).toDouble();
}
// Follow the background on quiet chunks: drop fast if it got quieter, rise slowly if louder.
final rate = rms < _pttNoiseFloor ? 0.2 : 0.05;
_pttNoiseFloor = (_pttNoiseFloor * (1 - rate) + rms * rate).clamp(_minNoiseFloor, _maxNoiseFloor).toDouble();
}
void _checkSpeechSilence() {
if (!_pttDetectorActive || _pttAutoStopTriggered || !_pttHasDetectedSpeech || pttStatus != PttStatus.recording || !liveSessionActive) {
if (!_pttDetectorActive || _pttAutoStopTriggered || pttStatus != PttStatus.recording || !liveSessionActive) {
return;
}
final now = DateTime.now();
if (!_pttHasDetectedSpeech) {
// User hasn't spoken since listening started: end the session instead of recording forever.
final startedAt = _pttListeningStartedAt;
if (startedAt != null && now.difference(startedAt) >= _noSpeechTimeout) {
_pttLog('live: no speech detected, stopping session');
_pttAutoStopTriggered = true;
liveSessionActive = false;
_stopLivePlayback();
cancelPushToTalk().then((_) {
pttErrorMessage = "Can't hear you, so the session was stopped. Tap the button to start again.";
notifyListeners();
}).catchError((_) {});
}
return;
}
final lastChunk = _pttLastAudioChunkAt;
if (lastChunk == null || now.difference(lastChunk) > _audioChunkTimeout) return;
final lastSpeech = _pttLastSpeechAt;
if (lastSpeech == null || now.difference(lastSpeech) < _silenceHoldDuration) return;
final startedAt = _pttListeningStartedAt;
final tooLong = startedAt != null && now.difference(startedAt) >= _maxUtteranceDuration;
if (!tooLong && (lastSpeech == null || now.difference(lastSpeech) < _silenceHoldDuration)) return;
_pttAutoStopTriggered = true;
_pttDetectorActive = false;
_pttSilenceTimer?.cancel();
_pttSilenceTimer = null;
_pttLog('live: silence after speech, auto-stopping recording');
_pttLog('live: ${tooLong ? 'max utterance length reached' : 'silence after speech'}, auto-stopping recording');
endPushToTalk().catchError((_) {});
}

28
lib/modules/copilot/screens/copilot_live_talk_page.dart
Unescape Escape View File

@ -1,10 +1,12 @@
import 'package:flutter/material.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:provider/provider.dart';
import 'package:test_sa/border_beam/src/beam_style.dart';
import 'package:test_sa/border_beam/src/border_beam.dart';
import 'package:test_sa/controllers/providers/api/user_provider.dart';
import 'package:test_sa/extensions/context_extension.dart';
import 'package:test_sa/extensions/int_extensions.dart';
import 'package:test_sa/extensions/string_extensions.dart';
import 'package:test_sa/extensions/text_extensions.dart';
import 'package:test_sa/extensions/widget_extensions.dart';
import 'package:test_sa/new_views/app_style/app_color.dart';
@ -33,6 +35,18 @@ class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> with WidgetsB
@override
void initState() {
super.initState();
makeConnection();
}
/// Microphone is only needed once the user starts the session, so ask on button tap.
Future<bool> _ensureMicPermission() async {
if (await Permission.microphone.isGranted) return true;
if ((await Permission.microphone.request()).isGranted) return true;
"Microphone permission denied. Please allow microphone access to use Live Session.".showToast;
return false;
}
void makeConnection() {
WidgetsBinding.instance.addObserver(this);
WidgetsBinding.instance.addPostFrameCallback((_) {
final provider = context.read<CopilotProvider>();
@ -89,14 +103,15 @@ class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> with WidgetsB
}
}
void _onMainButtonTap(CopilotProvider provider) {
// The button only starts/stops the whole session (never per-utterance), and
// stopping must work at any point (listening / thinking / speaking).
/// The button only starts/stops the whole session. Within a session the provider
/// auto-sends when the user stops speaking, plays the answer, then listens again.
Future<void> _onMainButtonTap(CopilotProvider provider) async {
if (provider.liveSessionActive) {
provider.stopLiveTalk().catchError((_) {});
} else {
provider.startLiveTalk(context.settingProvider.language!).catchError((_) {});
return;
}
if (!await _ensureMicPermission() || !mounted) return;
provider.startLiveTalk(context.settingProvider.language!).catchError((_) {});
}
Widget _buildConnectionPlaceholder(BuildContext context, CopilotProvider provider) {
@ -128,7 +143,7 @@ class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> with WidgetsB
@override
Widget build(BuildContext context) {
return Scaffold(
appBar: const DefaultAppBar(title: "Live Session"),
appBar: const DefaultAppBar(title: "Live Session", showCopilot: false),
body: Column(
children: [
Expanded(
@ -254,4 +269,3 @@ class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> with WidgetsB
);
}
}

Write Preview
Loading…
Cancel
Save