Haroon6138
/
cloudsolutions-atoms
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 28 Wiki Activity

live session implementation in progress

Browse Source
main_production_copilot
WaseemAbbasi22 4 days ago
parent 61d20d20fd
commit c6cefed7f4
2 changed files with 222 additions and 66 deletions
Show Stats Download Patch File Download Diff File

175
lib/modules/copilot/controller/copilot_provider.dart
Unescape Escape View File

@ -3,8 +3,10 @@ import 'dart:convert';
import 'dart:developer' as log1;
import 'dart:io';
import 'dart:math';
import 'dart:typed_data';
import 'package:signalr_netcore/ihub_protocol.dart';
import 'package:signalr_netcore/signalr_client.dart';
import 'package:audioplayers/audioplayers.dart' as audioplayers;
import 'package:flutter/foundation.dart';
import 'package:flutter_sound/flutter_sound.dart' as sound;
import 'package:http/http.dart';
@ -54,17 +56,30 @@ class CopilotProvider extends ChangeNotifier {
Future<void> _pttSendQueue = Future.value();
int _pttChunkCount = 0;
// Live Talk voice-activity (silence) detection.
StreamSubscription<sound.RecordingDisposition>? _pttProgressSub;
// Live Talk voice-activity (silence) detection, computed from the PCM mic
// chunks (onProgress metering is not emitted when recording to a stream).
DateTime? _pttLastLoudAt;
DateTime? _pttRecordingStartedAt;
bool _pttAutoStopTriggered = false;
bool _pttHasDetectedSpeech = false;
static const double _silenceDbThreshold = 38.0;
static const Duration _silenceHoldDuration = Duration(milliseconds: 1400);
// Adaptive noise floor (RMS) learned from quiet chunks so detection works
// regardless of device mic gain / ambient noise.
double _pttNoiseFloor = 0.01;
static const Duration _silenceHoldDuration = Duration(milliseconds: 3000);
CopilotMessage? _pttCurrentAssistantMsg;
// Live Talk: the provider plays the bot's TTS audio itself so it can resume
// listening only AFTER the bot has finished speaking (avoids recording the
// bot's own voice). Push-to-Talk does not use any of this.
final audioplayers.AudioPlayer _livePlayer = audioplayers.AudioPlayer();
final List<Uint8List> _liveAudioQueue = [];
bool _liveIsPlaying = false;
Timer? _liveRestartTimer;
CopilotProvider() {
_livePlayer.onPlayerComplete.listen((_) => _onLiveClipComplete());
}
void _pttLog(String message) => log1.log('[PTT] $message');
Future<void> loadHistory() async {
@ -611,6 +626,7 @@ class CopilotProvider extends ChangeNotifier {
/// Stops the current Live Talk session (does not auto-restart listening again).
Future<void> stopLiveTalk() async {
liveSessionActive = false;
_stopLivePlayback();
if (pttStatus == PttStatus.recording) {
await endPushToTalk();
} else {
@ -689,6 +705,7 @@ class CopilotProvider extends ChangeNotifier {
_pttChunkCount++;
_pttLog('mic chunk #$_pttChunkCount captured bytes=${chunk.length}');
_enqueueChunkSend(chunk);
if (_pttIsLive) _processSilenceDetection(chunk);
});
await _pttRecorder.startRecorder(
@ -707,41 +724,60 @@ class CopilotProvider extends ChangeNotifier {
}
}
/// Monitors mic decibel levels while in Live Talk mode and automatically
/// triggers [endPushToTalk] once the user has been silent for
/// [_silenceHoldDuration] AFTER having spoken, so the user doesn't need to
/// tap anything. Won't trigger on the initial silence before the user
/// starts speaking.
/// Resets voice-activity state for a new Live Talk recording. Actual silence
/// detection happens per mic chunk in [_processSilenceDetection].
void _startSilenceDetection() {
_pttAutoStopTriggered = false;
_pttHasDetectedSpeech = false;
_pttRecordingStartedAt = DateTime.now();
_pttLastLoudAt = null;
_pttProgressSub?.cancel();
_pttProgressSub = _pttRecorder.onProgress?.listen((disposition) {
final db = disposition.decibels ?? 0;
final now = DateTime.now();
if (db > _silenceDbThreshold) {
_pttLastLoudAt = now;
_pttHasDetectedSpeech = true;
return;
}
if (_pttAutoStopTriggered || !_pttHasDetectedSpeech) return;
final lastLoud = _pttLastLoudAt;
if (lastLoud != null && now.difference(lastLoud) >= _silenceHoldDuration) {
_pttAutoStopTriggered = true;
_pttLog('_startSilenceDetection: silence detected after speech, auto-stopping recording (live talk)');
endPushToTalk().catchError((_) {});
}
});
_pttNoiseFloor = 0.01;
}
void _stopSilenceDetection() {
_pttProgressSub?.cancel();
_pttProgressSub = null;
_pttLastLoudAt = null;
_pttRecordingStartedAt = null;
_pttHasDetectedSpeech = false;
_pttAutoStopTriggered = false;
}
/// Live Talk auto-stop: once the user has spoken and then stayed silent for
/// [_silenceHoldDuration] (3s), automatically send StopRecording — the button
/// is only used to start/stop the whole session, never per-utterance.
/// Uses an adaptive noise floor so it works across devices/mic gains.
void _processSilenceDetection(Uint8List chunk) {
if (_pttAutoStopTriggered) return;
final rms = _chunkRms(chunk);
final now = DateTime.now();
// Speaking if clearly above the learned noise floor (with an absolute min).
final speechThreshold = (_pttNoiseFloor * 2.2).clamp(0.008, 0.3);
final speaking = rms > speechThreshold;
_pttLog('silence: rms=${rms.toStringAsFixed(4)} floor=${_pttNoiseFloor.toStringAsFixed(4)} thr=${speechThreshold.toStringAsFixed(4)} speaking=$speaking spoke=$_pttHasDetectedSpeech');
if (speaking) {
_pttLastLoudAt = now;
_pttHasDetectedSpeech = true;
return;
}
// Quiet chunk: slowly adapt the noise floor toward the current level.
_pttNoiseFloor = _pttNoiseFloor * 0.9 + rms * 0.1;
if (!_pttHasDetectedSpeech) return;
final lastLoud = _pttLastLoudAt;
if (lastLoud != null && now.difference(lastLoud) >= _silenceHoldDuration) {
_pttAutoStopTriggered = true;
_pttLog('live: silence after speech, auto-stopping recording');
endPushToTalk().catchError((_) {});
}
}
/// Root-mean-square amplitude (0..1) of a PCM16 mono little-endian chunk.
double _chunkRms(Uint8List chunk) {
final sampleCount = chunk.length ~/ 2;
if (sampleCount == 0) return 0;
final data = ByteData.sublistView(chunk);
double sumSquares = 0;
for (var i = 0; i < sampleCount; i++) {
final sample = data.getInt16(i * 2, Endian.little) / 32768.0;
sumSquares += sample * sample;
}
return sqrt(sumSquares / sampleCount);
}
Future<void> _stopMicStreamingAndFlush() async {
@ -839,13 +875,17 @@ class CopilotProvider extends ChangeNotifier {
final text = data?['voiceAnswer']?.toString() ?? '';
_pttCurrentAssistantMsg = CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.assistant, content: text, timestamp: DateTime.now());
pttMessages.add(_pttCurrentAssistantMsg!);
pttStatus = PttStatus.idle;
notifyListeners();
if (liveSessionActive && _pttIsLive) {
final lang = _pttLanguage ?? '1';
_pttLog('ResponseReceived: live session active, auto-restarting listening');
Future.microtask(() => beginPushToTalk(lang, isLive: true).catchError((_) {}));
// Live Talk: let the bot finish speaking its TTS audio, then resume
// listening. _maybeRestartLiveListening is debounced so it waits for any
// audio clips that arrive right after this text response.
pttStatus = PttStatus.playingAudio;
notifyListeners();
_maybeRestartLiveListening();
} else {
pttStatus = PttStatus.idle;
notifyListeners();
}
}
@ -854,13 +894,69 @@ class CopilotProvider extends ChangeNotifier {
try {
final bytes = base64Decode(params.first.toString());
_pttLog('event AudioReceived bytes=${bytes.length}');
_pttCurrentAssistantMsg?.ttsAudioClips.add(bytes);
notifyListeners();
if (liveSessionActive && _pttIsLive) {
// Live Talk: queue and play through the provider so we know when the
// bot finished speaking (to resume listening). No widget player.
_liveAudioQueue.add(bytes);
_playNextLiveClip();
} else {
_pttCurrentAssistantMsg?.ttsAudioClips.add(bytes);
notifyListeners();
}
} catch (e) {
_pttLog('_onAudioReceived: failed to decode error=$e');
}
}
// ----- Live Talk audio playback + auto-resume listening -----
void _playNextLiveClip() {
if (_liveIsPlaying || _liveAudioQueue.isEmpty) return;
_liveRestartTimer?.cancel();
final clip = _liveAudioQueue.removeAt(0);
_liveIsPlaying = true;
if (pttStatus != PttStatus.playingAudio) {
pttStatus = PttStatus.playingAudio;
notifyListeners();
}
_livePlayer.play(audioplayers.BytesSource(clip, mimeType: 'audio/wav')).catchError((e) {
_pttLog('_playNextLiveClip: play error=$e');
_onLiveClipComplete();
});
}
void _onLiveClipComplete() {
_liveIsPlaying = false;
if (_liveAudioQueue.isNotEmpty) {
_playNextLiveClip();
} else {
_maybeRestartLiveListening();
}
}
/// After the bot stops speaking, resume listening. Debounced so trailing
/// audio clips (arriving just after) don't cause a premature restart.
void _maybeRestartLiveListening() {
if (!liveSessionActive || !_pttIsLive) return;
_liveRestartTimer?.cancel();
_liveRestartTimer = Timer(const Duration(milliseconds: 600), () {
if (!liveSessionActive || !_pttIsLive) return;
if (_liveIsPlaying || _liveAudioQueue.isNotEmpty) return;
if (pttStatus == PttStatus.recording || pttStatus == PttStatus.connecting) return;
_pttLog('live: bot finished speaking, resuming listening');
pttStatus = PttStatus.idle;
beginPushToTalk(_pttLanguage ?? '1', isLive: true).catchError((_) {});
});
}
void _stopLivePlayback() {
_liveRestartTimer?.cancel();
_liveRestartTimer = null;
_liveAudioQueue.clear();
_liveIsPlaying = false;
_livePlayer.stop().catchError((_) {});
}
void _onRecordingError(List<Object?>? params) {
final msg = (params != null && params.isNotEmpty) ? params.first?.toString() : null;
_pttLog('event RecordingError message=$msg');
@ -868,6 +964,7 @@ class CopilotProvider extends ChangeNotifier {
pttErrorMessage = msg ?? "Recording error occurred";
pttStatus = PttStatus.error;
liveSessionActive = false;
_stopLivePlayback();
_abortMicStreaming();
notifyListeners();
}
@ -887,6 +984,8 @@ class CopilotProvider extends ChangeNotifier {
void dispose() {
_streamSub?.cancel();
_cancelResponseWatchdog();
_liveRestartTimer?.cancel();
_livePlayer.dispose();
_abortMicStreaming();
if (_pttRecorderOpen) {
_pttRecorder.closeRecorder();

113
lib/modules/copilot/screens/copilot_live_talk_page.dart
Unescape Escape View File

@ -1,5 +1,7 @@
import 'package:flutter/material.dart';
import 'package:provider/provider.dart';
import 'package:test_sa/border_beam/src/beam_style.dart';
import 'package:test_sa/border_beam/src/border_beam.dart';
import 'package:test_sa/controllers/providers/api/user_provider.dart';
import 'package:test_sa/extensions/context_extension.dart';
import 'package:test_sa/extensions/int_extensions.dart';
@ -7,6 +9,7 @@ import 'package:test_sa/extensions/text_extensions.dart';
import 'package:test_sa/extensions/widget_extensions.dart';
import 'package:test_sa/new_views/app_style/app_color.dart';
import 'package:test_sa/new_views/common_widgets/default_app_bar.dart';
import 'package:test_sa/thinking_orbs/src/thinking_orb.dart';
import '../controller/copilot_provider.dart';
import '../widgets/copilot_message_bubble.dart';
@ -21,7 +24,7 @@ class CopilotLiveTalkPage extends StatefulWidget {
State<CopilotLiveTalkPage> createState() => _CopilotLiveTalkPageState();
}
class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> {
class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> with WidgetsBindingObserver {
final ScrollController _scrollController = ScrollController();
CopilotProvider? _provider;
@ -34,6 +37,7 @@ class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> {
@override
void initState() {
super.initState();
WidgetsBinding.instance.addObserver(this);
WidgetsBinding.instance.addPostFrameCallback((_) {
final provider = context.read<CopilotProvider>();
provider.startNewPttConversation();
@ -41,8 +45,19 @@ class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> {
});
}
@override
void didChangeAppLifecycleState(AppLifecycleState state) {
if (state == AppLifecycleState.paused) {
// OS suspends sockets/mic in background; stop the live session.
_provider?.stopLiveTalk().catchError((_) {});
} else if (state == AppLifecycleState.resumed) {
_provider?.ensurePushToTalkConnected().catchError((_) {});
}
}
@override
void dispose() {
WidgetsBinding.instance.removeObserver(this);
_scrollController.dispose();
_provider?.stopLiveTalk().catchError((_) {});
_provider?.disconnectPushToTalkHub().catchError((_) {});
@ -79,10 +94,8 @@ class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> {
}
void _onMainButtonTap(CopilotProvider provider) {
final status = provider.pttStatus;
final isBusy = status == PttStatus.connecting || status == PttStatus.thinking;
if (isBusy) return;
// The button only starts/stops the whole session (never per-utterance), and
// stopping must work at any point (listening / thinking / speaking).
if (provider.liveSessionActive) {
provider.stopLiveTalk().catchError((_) {});
} else {
@ -90,6 +103,35 @@ class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> {
}
}
/// Shown in place of the mic while the socket is connecting / disconnected.
Widget _buildConnectionPlaceholder(BuildContext context, CopilotProvider provider) {
// Treat "disconnected without an error" as connecting (e.g. the first frame
// before the connection attempt kicks off).
final isConnecting = provider.pttHubState == PttHubState.connecting || provider.pttErrorMessage == null;
return SizedBox(
height: 148,
child: Column(
mainAxisAlignment: MainAxisAlignment.center,
mainAxisSize: MainAxisSize.min,
children: [
if (isConnecting) ...[
const SizedBox(height: 28, width: 28, child: CircularProgressIndicator(strokeWidth: 2.5)),
12.height,
Text(
"Connecting…",
style: AppTextStyles.bodyText2.copyWith(color: AppColor.lightTextColor(context)),
),
] else
TextButton.icon(
onPressed: () => provider.ensurePushToTalkConnected().catchError((_) {}),
icon: const Icon(Icons.refresh_rounded),
label: const Text("Retry connection"),
),
],
),
).paddingOnly(top: 8, bottom: 8);
}
@override
Widget build(BuildContext context) {
return Scaffold(
@ -149,6 +191,10 @@ class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> {
SafeArea(
child: Consumer<CopilotProvider>(
builder: (context, provider, _) {
// Hide the mic until the socket connection is established.
if (provider.pttHubState != PttHubState.connected) {
return _buildConnectionPlaceholder(context, provider);
}
final status = provider.pttStatus;
final isActive = provider.liveSessionActive;
final isRecording = status == PttStatus.recording;
@ -162,30 +208,41 @@ class _CopilotLiveTalkPageState extends State<CopilotLiveTalkPage> {
style: AppTextStyles.bodyText2.copyWith(color: AppColor.lightTextColor(context)),
),
16.height,
GestureDetector(
onTap: isBusy ? null : () => _onMainButtonTap(provider),
child: AnimatedContainer(
duration: const Duration(milliseconds: 150),
height: isRecording || isActive ? 96 : 80,
width: isRecording || isActive ? 96 : 80,
decoration: BoxDecoration(
shape: BoxShape.circle,
color: isRecording
? Colors.red
: isActive
? AppColor.primary10.withValues(alpha: 0.8)
: AppColor.primary10,
Container(
height: 100,
alignment: Alignment.center,
child: Container(
padding: const EdgeInsets.all(0),
decoration: BoxDecoration(color: AppColor.background(context), shape: BoxShape.circle, border: Border.all(color: Colors.transparent)),
child: BorderBeam(
size: BeamSize.pulseOutside,
colorVariant: BeamColorVariant.colorful,
reserveGlowSpace: false,
borderRadius: 50,
strength: 0.7,
child: GestureDetector(
onTap: () => _onMainButtonTap(provider),
child: AnimatedContainer(
duration: const Duration(milliseconds: 150),
height: isRecording || isActive ? 96 : 80,
width: isRecording || isActive ? 96 : 80,
child: isBusy
? Padding(
padding: const EdgeInsets.all(12),
child: ThinkingOrb(
size: 48,
state: ThinkingOrbState.working,
theme: context.isDark ? ThinkingOrbTheme.dark : ThinkingOrbTheme.light,
),
)
: Icon(
isActive ? Icons.stop_rounded : Icons.graphic_eq_rounded,
color: AppColor.icon2Color(context),
size: isRecording || isActive ? 44 : 36,
),
),
),
),
child: isBusy
? const Padding(
padding: EdgeInsets.all(24),
child: CircularProgressIndicator(color: Colors.white, strokeWidth: 2.5),
)
: Icon(
isActive ? Icons.stop_rounded : Icons.graphic_eq_rounded,
color: Colors.white,
size: isRecording || isActive ? 44 : 36,
),
),
),
8.height,

Write Preview
Loading…
Cancel
Save