Haroon6138
/
cloudsolutions-atoms
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 28 Wiki Activity

live session improvement

Browse Source
main_production_copilot
WaseemAbbasi22 1 day ago
parent c6cefed7f4
commit f1be1a6d7e
2 changed files with 103 additions and 50 deletions
Show Stats Download Patch File Download Diff File

143
lib/modules/copilot/controller/copilot_provider.dart
Unescape Escape View File

@ -61,9 +61,15 @@ class CopilotProvider extends ChangeNotifier {
DateTime? _pttLastLoudAt;
bool _pttAutoStopTriggered = false;
bool _pttHasDetectedSpeech = false;
// Adaptive noise floor (RMS) learned from quiet chunks so detection works
// regardless of device mic gain / ambient noise.
// Self-calibrating VAD: we continuously track the quiet "floor" AND the loud
// speech "peak", then put the speaking threshold in the gap between them, so
// detection adapts to any device mic gain / ambient noise level.
double _pttNoiseFloor = 0.01;
double _pttSpeechPeak = 0.05;
// Smoothed voice-activity (0..1 moving average of "is this chunk speech?").
// Brief room-noise/echo spikes can't reset the silence timer — only sustained
// speech keeps this high, which is what we key the auto-stop off of.
double _pttVoiceActivity = 0.0;
static const Duration _silenceHoldDuration = Duration(milliseconds: 3000);
CopilotMessage? _pttCurrentAssistantMsg;
@ -392,6 +398,38 @@ class CopilotProvider extends ChangeNotifier {
await _stopHubAndCleanup();
_setHubState(PttHubState.connecting);
// signalr_netcore can throw a "Null check operator used on a null value"
// from HubConnection._startInternal when the server closes the long-poll
// connection mid-handshake (a package race on _handshakeCompleter). It is
// transient, so retry a few times before surfacing an error.
const maxAttempts = 3;
Object? lastError;
for (var attempt = 1; attempt <= maxAttempts; attempt++) {
try {
await _buildAndStartHub();
return; // connected
} catch (e, st) {
lastError = e;
_pttLog('connectPushToTalkHub: attempt $attempt/$maxAttempts failed error=$e');
await _stopHubAndCleanup();
if (attempt < maxAttempts) {
await Future.delayed(Duration(milliseconds: 400 * attempt));
} else {
_pttLog('connectPushToTalkHub: giving up\n$st');
}
}
}
pttErrorMessage = "Failed to connect to push-to-talk service";
pttStatus = PttStatus.error;
pttHubState = PttHubState.disconnected;
notifyListeners();
throw lastError ?? StateError('Failed to connect');
}
/// Builds a single fresh hub and starts it. Throws on failure so the caller
/// ([_doConnectPushToTalkHub]) can retry.
Future<void> _buildAndStartHub() async {
final token = ApiManager.instance.user?.token ?? '';
final url = "${URLs.copilotHubUrl}?access_token=$token";
_pttLog('connectPushToTalkHub: building connection url=${URLs.copilotHubUrl}');
@ -442,33 +480,23 @@ class CopilotProvider extends ChangeNotifier {
_handleConnectionInterrupted("Connection lost. Tap retry to reconnect.");
});
_pttHub!.on("RecordingStarted", _onRecordingStarted);
_pttHub!.on("RecordingStopped", _onRecordingStopped);
_pttHub!.on("RecordingCancelled", _onRecordingCancelled);
_pttHub!.on("TranscriptionReceived", _onTranscriptionReceived);
_pttHub!.on("ResponseReceived", _onResponseReceived);
_pttHub!.on("AudioReceived", _onAudioReceived);
_pttHub!.on("RecordingError", _onRecordingError);
try {
_pttLog('connectPushToTalkHub: calling start()...');
await _pttHub!.start()?.timeout(const Duration(seconds: 20), onTimeout: () {
throw TimeoutException('Connection to push-to-talk service timed out');
});
_pttLog('connectPushToTalkHub: connected, state=${_pttHub!.state}');
pttHubState = PttHubState.connected;
if (pttStatus == PttStatus.error) pttStatus = PttStatus.idle;
pttErrorMessage = null;
notifyListeners();
} catch (e, st) {
_pttLog('connectPushToTalkHub: failed error=$e\n$st');
pttErrorMessage = "Failed to connect to push-to-talk service";
pttStatus = PttStatus.error;
await _stopHubAndCleanup();
pttHubState = PttHubState.disconnected;
notifyListeners();
rethrow;
}
hub.on("RecordingStarted", _onRecordingStarted);
hub.on("RecordingStopped", _onRecordingStopped);
hub.on("RecordingCancelled", _onRecordingCancelled);
hub.on("TranscriptionReceived", _onTranscriptionReceived);
hub.on("ResponseReceived", _onResponseReceived);
hub.on("AudioReceived", _onAudioReceived);
hub.on("RecordingError", _onRecordingError);
_pttLog('connectPushToTalkHub: calling start()...');
await hub.start()?.timeout(const Duration(seconds: 20), onTimeout: () {
throw TimeoutException('Connection to push-to-talk service timed out');
});
_pttLog('connectPushToTalkHub: connected, state=${hub.state}');
pttHubState = PttHubState.connected;
if (pttStatus == PttStatus.error) pttStatus = PttStatus.idle;
pttErrorMessage = null;
notifyListeners();
}
void _setHubState(PttHubState state) {
@ -623,15 +651,17 @@ class CopilotProvider extends ChangeNotifier {
await beginPushToTalk(language, isLive: true);
}
/// Stops the current Live Talk session (does not auto-restart listening again).
/// Stops the current Live Talk session completely. The button is only used to
/// start/stop the whole session, so ending it must hard-stop everything:
/// no more listening, thinking, or speaking until the user taps to start again.
Future<void> stopLiveTalk() async {
_pttLog('stopLiveTalk: ending session, stopping everything');
liveSessionActive = false;
_pttIsLive = false;
_pttAutoStopTriggered = true; // block any in-flight auto-stop
_cancelResponseWatchdog();
_stopLivePlayback();
if (pttStatus == PttStatus.recording) {
await endPushToTalk();
} else {
await cancelPushToTalk();
}
await cancelPushToTalk();
}
Future<void> endPushToTalk() async {
@ -731,33 +761,60 @@ class CopilotProvider extends ChangeNotifier {
_pttHasDetectedSpeech = false;
_pttLastLoudAt = null;
_pttNoiseFloor = 0.01;
_pttSpeechPeak = 0.05;
_pttVoiceActivity = 0.0;
}
void _stopSilenceDetection() {
_pttLastLoudAt = null;
_pttHasDetectedSpeech = false;
_pttAutoStopTriggered = false;
_pttVoiceActivity = 0.0;
}
/// Live Talk auto-stop: once the user has spoken and then stayed silent for
/// [_silenceHoldDuration] (3s), automatically send StopRecording — the button
/// is only used to start/stop the whole session, never per-utterance.
/// Uses an adaptive noise floor so it works across devices/mic gains.
///
/// Self-calibrating + noise/echo tolerant:
/// - Tracks the quiet floor (fast down / slow up) and the speech peak
/// (fast up / slow down), and sets a per-chunk threshold 45% of the way
/// from floor toward peak.
/// - Smooths that per-chunk decision into [_pttVoiceActivity] so a few
/// stray room-noise / echo chunks can't masquerade as speech and keep
/// resetting the silence countdown. Only *sustained* speech counts.
void _processSilenceDetection(Uint8List chunk) {
if (_pttAutoStopTriggered) return;
final rms = _chunkRms(chunk);
final now = DateTime.now();
// Speaking if clearly above the learned noise floor (with an absolute min).
final speechThreshold = (_pttNoiseFloor * 2.2).clamp(0.008, 0.3);
final speaking = rms > speechThreshold;
_pttLog('silence: rms=${rms.toStringAsFixed(4)} floor=${_pttNoiseFloor.toStringAsFixed(4)} thr=${speechThreshold.toStringAsFixed(4)} speaking=$speaking spoke=$_pttHasDetectedSpeech');
if (speaking) {
if (rms < _pttNoiseFloor) {
_pttNoiseFloor = rms;
} else {
_pttNoiseFloor = _pttNoiseFloor * 0.995 + rms * 0.005;
}
if (rms > _pttSpeechPeak) {
_pttSpeechPeak = rms;
} else {
_pttSpeechPeak = _pttSpeechPeak * 0.995 + rms * 0.005;
}
final span = _pttSpeechPeak - _pttNoiseFloor;
final threshold = max(_pttNoiseFloor + span * 0.45, 0.012);
final chunkIsSpeech = rms > threshold && span > 0.01;
// Smooth: a single loud chunk only nudges activity, sustained speech climbs.
_pttVoiceActivity = _pttVoiceActivity * 0.8 + (chunkIsSpeech ? 1.0 : 0.0) * 0.2;
// Hysteresis: need clear sustained energy to count as actively speaking.
final activelySpeaking = _pttVoiceActivity > 0.4;
_pttLog('silence: rms=${rms.toStringAsFixed(4)} floor=${_pttNoiseFloor.toStringAsFixed(4)} peak=${_pttSpeechPeak.toStringAsFixed(4)} thr=${threshold.toStringAsFixed(4)} act=${_pttVoiceActivity.toStringAsFixed(2)} speaking=$activelySpeaking spoke=$_pttHasDetectedSpeech');
if (activelySpeaking) {
_pttLastLoudAt = now;
_pttHasDetectedSpeech = true;
return;
}
// Quiet chunk: slowly adapt the noise floor toward the current level.
_pttNoiseFloor = _pttNoiseFloor * 0.9 + rms * 0.1;
if (!_pttHasDetectedSpeech) return;
final lastLoud = _pttLastLoudAt;
if (lastLoud != null && now.difference(lastLoud) >= _silenceHoldDuration) {

10
lib/modules/copilot/screens/copilot_live_talk_page.dart
Unescape Escape View File

@ -13,10 +13,6 @@ import 'package:test_sa/thinking_orbs/src/thinking_orb.dart';
import '../controller/copilot_provider.dart';
import '../widgets/copilot_message_bubble.dart';
/// Live Talk realtime voice screen for Copilot.
/// Unlike Push-to-Talk (press & hold), this is a continuous conversation mode:
/// tap once to start, the app keeps listening/responding automatically until
/// the user taps again to stop.
class CopilotLiveTalkPage extends StatefulWidget {
const CopilotLiveTalkPage({Key? key}) : super(key: key);
@ -79,7 +75,7 @@ class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> with WidgetsB
String _statusLabel(PttStatus status, bool isLiveActive) {
switch (status) {
case PttStatus.idle:
return isLiveActive ? "Getting ready…" : "Tap to start Live Talk";
return isLiveActive ? "Getting ready…" : "Tap to start Live Session";
case PttStatus.connecting:
return "Connecting…";
case PttStatus.recording:
@ -135,7 +131,7 @@ class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> with WidgetsB
@override
Widget build(BuildContext context) {
return Scaffold(
appBar: const DefaultAppBar(title: "Live Talk"),
appBar: const DefaultAppBar(title: "Live Session"),
body: Column(
children: [
Expanded(
@ -248,7 +244,7 @@ class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> with WidgetsB
8.height,
if (isActive)
Text(
"Tap to stop",
"Tap to stop session",
style: AppTextStyles.bodyText2.copyWith(color: AppColor.lightTextColor(context)),
),
24.height,

Write Preview
Loading…
Cancel
Save