import 'dart:async'; import 'dart:convert'; import 'dart:developer' as log1; import 'dart:io'; import 'dart:math'; import 'dart:typed_data'; import 'package:signalr_netcore/ihub_protocol.dart'; import 'package:signalr_netcore/signalr_client.dart'; import 'package:audioplayers/audioplayers.dart' as audioplayers; import 'package:flutter/foundation.dart'; import 'package:flutter_sound/flutter_sound.dart' as sound; import 'package:http/http.dart'; import 'package:permission_handler/permission_handler.dart'; import 'package:test_sa/controllers/api_routes/api_manager.dart'; import 'package:test_sa/controllers/api_routes/urls.dart'; import '../model/copilot_message_model.dart'; import '../utils/ptt_signalr_http_client.dart'; enum PttStatus { idle, connecting, recording, thinking, playingAudio, error } /// Connection state of the push-to-talk SignalR hub (observable by the UI). enum PttHubState { disconnected, connecting, connected } class CopilotProvider extends ChangeNotifier { List history = []; bool historyLoading = false; List messages = []; bool conversationLoading = false; bool isStreaming = false; bool isTranscribing = false; String? sessionId; String? errorMessage; StreamSubscription? _streamSub; HubConnection? _pttHub; PttSignalRHttpClient? _pttHttpClient; // Serializes connect/disconnect so they never overlap when the user navigates // in and out of the Push-to-Talk screen quickly. Future _pttOpQueue = Future.value(); PttStatus pttStatus = PttStatus.idle; PttHubState pttHubState = PttHubState.disconnected; String? pttErrorMessage; List pttMessages = []; String? pttSessionId; bool _pttButtonHeld = false; bool _pttIsLive = false; String? _pttLanguage; bool liveSessionActive = false; final sound.FlutterSoundRecorder _pttRecorder = sound.FlutterSoundRecorder(); bool _pttRecorderOpen = false; StreamController? _pttMicStreamController; StreamSubscription? _pttMicSub; Future _pttSendQueue = Future.value(); int _pttChunkCount = 0; // Live Talk voice-activity (silence) detection, computed from the PCM mic // chunks (onProgress metering is not emitted when recording to a stream). DateTime? _pttLastLoudAt; bool _pttAutoStopTriggered = false; bool _pttHasDetectedSpeech = false; // Self-calibrating VAD: we continuously track the quiet "floor" AND the loud // speech "peak", then put the speaking threshold in the gap between them, so // detection adapts to any device mic gain / ambient noise level. double _pttNoiseFloor = 0.01; double _pttSpeechPeak = 0.05; // Smoothed voice-activity (0..1 moving average of "is this chunk speech?"). // Brief room-noise/echo spikes can't reset the silence timer — only sustained // speech keeps this high, which is what we key the auto-stop off of. double _pttVoiceActivity = 0.0; static const Duration _silenceHoldDuration = Duration(milliseconds: 3000); CopilotMessage? _pttCurrentAssistantMsg; // Live Talk: the provider plays the bot's TTS audio itself so it can resume // listening only AFTER the bot has finished speaking (avoids recording the // bot's own voice). Push-to-Talk does not use any of this. final audioplayers.AudioPlayer _livePlayer = audioplayers.AudioPlayer(); final List _liveAudioQueue = []; bool _liveIsPlaying = false; Timer? _liveRestartTimer; CopilotProvider() { _livePlayer.onPlayerComplete.listen((_) => _onLiveClipComplete()); } void _pttLog(String message) => log1.log('[PTT] $message'); Future loadHistory() async { historyLoading = true; errorMessage = null; notifyListeners(); try { final res = await ApiManager.instance.get( URLs.copilotGetQueryHistory, enableToastMessage: false, ); final decoded = jsonDecode(res.body); if (decoded is Map && decoded['isSuccess'] == true) { final List threads = decoded['data']?['threads'] ?? []; history = threads.map((e) => CopilotHistoryItem.fromJson(e)).toList(); } else { errorMessage = (decoded is Map ? decoded['message'] : null) ?? "Failed to load history"; } } catch (e) { errorMessage = "Failed to load history"; } finally { historyLoading = false; notifyListeners(); } } Future deleteHistory(String threadId) async { final prevList = List.from(history); try { final res = await ApiManager.instance.delete(URLs.copilotDeleteQueryHistory(threadId), body: const {}); final decoded = jsonDecode(res.body); final success = decoded is Map && decoded['isSuccess'] == true && decoded['data'] == true; if (success) { history.removeWhere((h) => h.threadId == threadId); notifyListeners(); } else { history = prevList; errorMessage = (decoded is Map ? decoded['message'] : null) ?? "Failed to delete"; notifyListeners(); } } catch (e) { history = prevList; errorMessage = "Failed to delete"; notifyListeners(); } } Future loadConversation(String threadId) async { conversationLoading = true; errorMessage = null; messages = []; sessionId = threadId; notifyListeners(); try { final res = await ApiManager.instance.get( URLs.copilotGetConversationByThreadId(threadId), enableToastMessage: false, ); final decoded = jsonDecode(res.body); if (decoded is Map && decoded['isSuccess'] == true) { final List rawMessages = decoded['data']?['messages'] ?? []; messages = rawMessages.map((e) => CopilotMessage.fromHistoryJson(e)).toList(); } else { errorMessage = (decoded is Map ? decoded['message'] : null) ?? "Failed to load conversation"; } } catch (e) { errorMessage = "Failed to load conversation"; } finally { conversationLoading = false; notifyListeners(); } } void startNewConversation() { _streamSub?.cancel(); sessionId = null; messages = []; errorMessage = null; isStreaming = false; notifyListeners(); } Future sendText(String query, String language, File? image) async { if (isStreaming || query.trim().isEmpty) return; sessionId ??= _generateSessionId(); messages.add(CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.user, content: query, timestamp: DateTime.now())); final assistantMsg = CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.assistant, isStreaming: true, timestamp: DateTime.now()); messages.add(assistantMsg); isStreaming = true; errorMessage = null; notifyListeners(); try { final streamedResponse = await ApiManager.instance.postFormDataStreaming( URLs.copilotQueryStream, body: { "sessionId": sessionId!, "isImage": "${image != null}", "isAudio": "false", "langId": "$language", 'isFromMobile': 'true', "query": query, }, file: image, ); await _consumeStream(streamedResponse, assistantMsg, null); } catch (e) { _onStreamError(assistantMsg, e); } } Future sendAudio(File audioFile, String language) async { if (isStreaming) return; sessionId ??= _generateSessionId(); final humanAudioMsg = CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.user, localAudioPath: audioFile.path, timestamp: DateTime.now()); messages.add(humanAudioMsg); final assistantMsg = CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.assistant, isStreaming: true, timestamp: DateTime.now()); messages.add(assistantMsg); isStreaming = true; errorMessage = null; notifyListeners(); try { final streamedResponse = await ApiManager.instance.postFormDataStreamingWithFile( URLs.copilotQueryStream, fields: { "sessionId": sessionId!, "isAudio": "true", "langId": "$language", 'isFromMobile': 'true', }, fileFieldName: "AudioFile", file: audioFile, filename: audioFile.path.split('/').last, ); await _consumeStream(streamedResponse, assistantMsg, humanAudioMsg); } catch (e) { _onStreamError(assistantMsg, e); } } Future transcribeAudio(File audioFile, String language) async { isTranscribing = true; errorMessage = null; notifyListeners(); final exists = await audioFile.exists(); final sizeBytes = exists ? await audioFile.length() : 0; if (!exists || sizeBytes == 0) { errorMessage = "Recording file is empty or missing"; isTranscribing = false; notifyListeners(); return null; } try { final response = await ApiManager.instance.multiPart( URLs.copilotAudioTranscription, body: { "langId": language, }, showToast: false, files: [ MultipartFile.fromPath("AudioFile", audioFile.path, filename: audioFile.path.split('/').last), ], ); if (response.statusCode < 200 || response.statusCode >= 300) { errorMessage = "Failed to transcribe audio"; return null; } return _extractTranscriptionText(response.body); } catch (e) { errorMessage = "Failed to transcribe audio"; return null; } finally { isTranscribing = false; notifyListeners(); } } String? _extractTranscriptionText(String rawBody) { final trimmedRaw = rawBody.trim(); if (trimmedRaw.isEmpty) return null; try { final decoded = jsonDecode(trimmedRaw); if (decoded is String) { return decoded.trim(); } else if (decoded is Map) { final map = Map.from(decoded); final lowerKeyMap = {for (final entry in map.entries) entry.key.toLowerCase(): entry.value}; final text = lowerKeyMap['transcription'] ?? lowerKeyMap['data'] ?? lowerKeyMap['text'] ?? lowerKeyMap['result']; if (text is Map) { return _extractTranscriptionText(jsonEncode(text)); } if (text != null) return text.toString().trim(); } } catch (_) {} return trimmedRaw; } String _generateSessionId() { final rand = Random(); final bytes = List.generate(16, (_) => rand.nextInt(256)); bytes[6] = (bytes[6] & 0x0F) | 0x40; bytes[8] = (bytes[8] & 0x3F) | 0x80; String byteHex(int i) => bytes[i].toRadixString(16).padLeft(2, '0'); final hex = List.generate(16, byteHex).join(); return '${hex.substring(0, 8)}-${hex.substring(8, 12)}-${hex.substring(12, 16)}-${hex.substring(16, 20)}-${hex.substring(20, 32)}'; } Future _consumeStream(dynamic streamedResponse, CopilotMessage assistantMsg, CopilotMessage? humanAudioMsg) async { if (streamedResponse.statusCode < 200 || streamedResponse.statusCode >= 300) { _onStreamError(assistantMsg, "Server error ${streamedResponse.statusCode}"); return; } String buffer = ''; _streamSub = streamedResponse.stream.transform(utf8.decoder).listen( (rawChunk) { buffer += rawChunk; while (buffer.contains('\n\n')) { final idx = buffer.indexOf('\n\n'); final rawEvent = buffer.substring(0, idx); buffer = buffer.substring(idx + 2); _handleSseEvent(rawEvent, assistantMsg, humanAudioMsg); } }, onDone: () { if (buffer.trim().isNotEmpty) { _handleSseEvent(buffer, assistantMsg, humanAudioMsg); } assistantMsg.isStreaming = false; isStreaming = false; notifyListeners(); }, onError: (e) => _onStreamError(assistantMsg, e), cancelOnError: true, ); } void _handleSseEvent(String rawEvent, CopilotMessage assistantMsg, CopilotMessage? humanAudioMsg) { final dataLines = rawEvent.split('\n').where((line) => line.startsWith('data:')).map((line) => line.substring(5).trim()).toList(); if (dataLines.isEmpty) return; final payload = dataLines.join('\n'); if (payload == '[DONE]') { assistantMsg.isStreaming = false; isStreaming = false; notifyListeners(); return; } try { final json = jsonDecode(payload); final event = CopilotStreamEvent.fromJson(json); if (event.type == 'response' && event.content != null) { assistantMsg.content = event.content!.answer ?? ''; assistantMsg.documentAnswer = event.content!.documentAnswer; assistantMsg.websiteAnswer = event.content!.websiteAnswer; assistantMsg.images = event.content!.images; } else if (event.type == 'error') { _onStreamError(assistantMsg, event.content?.answer ?? 'Something went wrong'); return; } if (event.type == 'transcription' && event.transcriptionContent != null) { humanAudioMsg?.content = event.transcriptionContent ?? ''; humanAudioMsg?.localAudioPath = null; } notifyListeners(); } catch (_) {} } void _onStreamError(CopilotMessage assistantMsg, Object e) { assistantMsg.isStreaming = false; assistantMsg.isError = true; if (assistantMsg.content.isEmpty) { assistantMsg.content = "Something went wrong. Please try again."; } isStreaming = false; errorMessage = "Failed to get a response"; notifyListeners(); } void cancelStreaming() { _streamSub?.cancel(); isStreaming = false; if (messages.isNotEmpty && messages.last.role == CopilotRole.assistant) { messages.last.isStreaming = false; } } /// Builds a brand-new push-to-talk connection. Any previous connection is /// torn down first, so entering the screen always gives a fresh connection. /// Serialized via [_pttOpQueue] so it never races with a disconnect. Future connectPushToTalkHub() { _pttOpQueue = _pttOpQueue.then((_) => _doConnectPushToTalkHub()).catchError((e) { _pttLog('connectPushToTalkHub: queued error=$e'); }); return _pttOpQueue; } Future _doConnectPushToTalkHub() async { // Always start from a clean slate so every entry is a fresh connection. await _stopHubAndCleanup(); _setHubState(PttHubState.connecting); // signalr_netcore can throw a "Null check operator used on a null value" // from HubConnection._startInternal when the server closes the long-poll // connection mid-handshake (a package race on _handshakeCompleter). It is // transient, so retry a few times before surfacing an error. const maxAttempts = 3; Object? lastError; for (var attempt = 1; attempt <= maxAttempts; attempt++) { try { await _buildAndStartHub(); return; // connected } catch (e, st) { lastError = e; _pttLog('connectPushToTalkHub: attempt $attempt/$maxAttempts failed error=$e'); await _stopHubAndCleanup(); if (attempt < maxAttempts) { await Future.delayed(Duration(milliseconds: 400 * attempt)); } else { _pttLog('connectPushToTalkHub: giving up\n$st'); } } } pttErrorMessage = "Failed to connect to push-to-talk service"; pttStatus = PttStatus.error; pttHubState = PttHubState.disconnected; notifyListeners(); throw lastError ?? StateError('Failed to connect'); } /// Builds a single fresh hub and starts it. Throws on failure so the caller /// ([_doConnectPushToTalkHub]) can retry. Future _buildAndStartHub() async { final token = ApiManager.instance.user?.token ?? ''; final url = "${URLs.copilotHubUrl}?access_token=$token"; _pttLog('connectPushToTalkHub: building connection url=${URLs.copilotHubUrl}'); final httpClient = PttSignalRHttpClient(); _pttHttpClient = httpClient; final httpOp = HttpConnectionOptions( // Server only accepts LongPolling. transport: HttpTransportType.LongPolling, skipNegotiation: false, // Never log message content: it contains base64 audio and is very heavy. logMessageContent: false, headers: MessageHeaders()..setHeaderValue('Authorization', 'Bearer $token'), // Ensures the Authorization header is also sent on poll + the DELETE // that tells the server to drop the connection when we stop. accessTokenFactory: () async => token, // Closes sockets properly and truly aborts the pending long-poll on stop // (signalr_netcore's default client leaks them). httpClient: httpClient, ); final hub = HubConnectionBuilder().withUrl(url, options: httpOp).withAutomaticReconnect(retryDelays: [0, 2000, 5000, 10000, 20000]).build(); // More tolerant timeouts for mobile networks (defaults are 30s / 15s). hub.serverTimeoutInMilliseconds = 60000; hub.keepAliveIntervalInMilliseconds = 15000; _pttHub = hub; hub.onreconnecting(({Exception? error}) { if (!identical(hub, _pttHub)) return; _pttLog('hub reconnecting error=$error'); _setHubState(PttHubState.connecting); _handleConnectionInterrupted("Connection lost. Reconnecting…"); }); hub.onreconnected(({String? connectionId}) { if (!identical(hub, _pttHub)) return; _pttLog('hub reconnected connectionId=$connectionId'); pttHubState = PttHubState.connected; // Server-side recording state is bound to the old connection id, so the // user must start a new turn. if (pttStatus == PttStatus.error || pttStatus == PttStatus.connecting) { pttStatus = PttStatus.idle; } pttErrorMessage = null; notifyListeners(); }); hub.onclose(({Exception? error}) { if (!identical(hub, _pttHub)) return; // intentional stop of an old hub _pttLog('hub closed error=$error'); pttHubState = PttHubState.disconnected; _handleConnectionInterrupted("Connection lost. Tap retry to reconnect."); }); hub.on("RecordingStarted", _onRecordingStarted); hub.on("RecordingStopped", _onRecordingStopped); hub.on("RecordingCancelled", _onRecordingCancelled); hub.on("TranscriptionReceived", _onTranscriptionReceived); hub.on("ResponseReceived", _onResponseReceived); hub.on("AudioReceived", _onAudioReceived); hub.on("RecordingError", _onRecordingError); _pttLog('connectPushToTalkHub: calling start()...'); await hub.start()?.timeout(const Duration(seconds: 20), onTimeout: () { throw TimeoutException('Connection to push-to-talk service timed out'); }); _pttLog('connectPushToTalkHub: connected, state=${hub.state}'); pttHubState = PttHubState.connected; if (pttStatus == PttStatus.error) pttStatus = PttStatus.idle; pttErrorMessage = null; notifyListeners(); } void _setHubState(PttHubState state) { if (pttHubState == state) return; pttHubState = state; notifyListeners(); } /// Disconnects the push-to-talk connection cleanly. Called when leaving the /// screen. Serialized via [_pttOpQueue] so it never races with a connect. Future disconnectPushToTalkHub() { _pttOpQueue = _pttOpQueue.then((_) => _doDisconnectPushToTalkHub()).catchError((e) { _pttLog('disconnectPushToTalkHub: queued error=$e'); }); return _pttOpQueue; } Future _doDisconnectPushToTalkHub() async { _pttLog('disconnectPushToTalkHub: stopping'); await _stopHubAndCleanup(); pttStatus = PttStatus.idle; } /// Stops mic streaming and tears down the hub. Does NOT touch [_pttOpQueue] /// so it is safe to call from inside queued operations. Future _stopHubAndCleanup() async { _cancelResponseWatchdog(); // Detach first so callbacks (onclose) from this hub are ignored. final hub = _pttHub; _pttHub = null; final httpClient = _pttHttpClient; _pttHttpClient = null; pttHubState = PttHubState.disconnected; try { await _abortMicStreaming(); if (hub != null) { await hub.stop().timeout(const Duration(seconds: 10), onTimeout: () { _pttLog('_stopHubAndCleanup: stop() timed out, forcing cleanup'); }); } } catch (e) { _pttLog('_stopHubAndCleanup: error=$e'); } finally { // Kill any request still open (e.g. a pending long-poll) so no orphan // connections pile up between visits to the screen. httpClient?.closeAll(); } } /// Called when the hub drops (reconnecting/closed). Stops the mic and frees /// the UI so it never stays stuck in recording/thinking forever. void _handleConnectionInterrupted(String message) { _cancelResponseWatchdog(); _pttButtonHeld = false; liveSessionActive = false; _abortMicStreaming().catchError((_) {}); if (pttStatus != PttStatus.idle) { pttStatus = PttStatus.error; pttErrorMessage = message; } notifyListeners(); } /// Re-establishes the connection if it was lost (e.g. after the app was /// in background). Safe to call any time. Future ensurePushToTalkConnected() async { final state = _pttHub?.state; if (state == HubConnectionState.Connected || state == HubConnectionState.Connecting || state == HubConnectionState.Reconnecting) return; await connectPushToTalkHub(); if (_pttHub?.state == HubConnectionState.Connected && pttStatus == PttStatus.error) { pttStatus = PttStatus.idle; pttErrorMessage = null; notifyListeners(); } } Timer? _pttResponseWatchdog; static const Duration _responseTimeout = Duration(seconds: 60); void _startResponseWatchdog() { _cancelResponseWatchdog(); _pttResponseWatchdog = Timer(_responseTimeout, () { if (pttStatus != PttStatus.thinking) return; _pttLog('response watchdog fired: no response from server'); pttStatus = PttStatus.error; pttErrorMessage = "No response from server. Please try again."; liveSessionActive = false; notifyListeners(); }); } void _cancelResponseWatchdog() { _pttResponseWatchdog?.cancel(); _pttResponseWatchdog = null; } void startNewPttConversation() { pttSessionId = null; pttMessages = []; pttErrorMessage = null; pttStatus = PttStatus.idle; liveSessionActive = false; _pttIsLive = false; notifyListeners(); } Future beginPushToTalk(String language, {bool isLive = false}) async { if (pttStatus != PttStatus.idle && pttStatus != PttStatus.error) return; pttSessionId ??= _generateSessionId(); _pttButtonHeld = true; _pttIsLive = isLive; _pttLanguage = language; pttErrorMessage = null; pttStatus = PttStatus.connecting; notifyListeners(); try { if (_pttHub == null || _pttHub!.state != HubConnectionState.Connected) { await connectPushToTalkHub(); } final hub = _pttHub; if (hub == null || hub.state != HubConnectionState.Connected) { throw StateError('Push-to-talk hub is not connected'); } final args = [ { "sessionId": pttSessionId ?? '', "userId": ApiManager.instance.user?.id ?? '', "assetGroupId": (ApiManager.instance.assetGroup?.id ?? '').toString(), "languageId": "1", "isLiveSession": isLive, } ]; _pttLog('beginPushToTalk: invoking StartRecording args=$args'); await _pttHub!.invoke("StartRecording", args: args); _pttLog('beginPushToTalk: StartRecording invoked, waiting for RecordingStarted event'); } catch (e, st) { _pttLog('beginPushToTalk: failed error=$e\n$st'); pttStatus = PttStatus.error; pttErrorMessage = "Failed to start recording"; _pttButtonHeld = false; liveSessionActive = false; notifyListeners(); } } /// Starts a continuous "Live Talk" session: records, waits for a response, /// then automatically starts listening again until [stopLiveTalk] is called. Future startLiveTalk(String language) async { if (pttStatus != PttStatus.idle && pttStatus != PttStatus.error) return; liveSessionActive = true; await beginPushToTalk(language, isLive: true); } /// Stops the current Live Talk session completely. The button is only used to /// start/stop the whole session, so ending it must hard-stop everything: /// no more listening, thinking, or speaking until the user taps to start again. Future stopLiveTalk() async { _pttLog('stopLiveTalk: ending session, stopping everything'); liveSessionActive = false; _pttIsLive = false; _pttAutoStopTriggered = true; // block any in-flight auto-stop _cancelResponseWatchdog(); _stopLivePlayback(); await cancelPushToTalk(); } Future endPushToTalk() async { _pttButtonHeld = false; if (pttStatus != PttStatus.recording) { notifyListeners(); return; } pttStatus = PttStatus.thinking; notifyListeners(); _pttLog('endPushToTalk: stopping mic and flushing $_pttChunkCount chunks sent so far'); try { await _stopMicStreamingAndFlush(); if (_pttHub != null && _pttHub!.state == HubConnectionState.Connected) { _pttLog('endPushToTalk: invoking StopRecording'); await _pttHub!.invoke("StopRecording"); _startResponseWatchdog(); } else { throw StateError('Connection lost before StopRecording'); } } catch (e) { _pttLog('endPushToTalk: failed error=$e'); pttStatus = PttStatus.error; pttErrorMessage = "Failed to stop recording"; notifyListeners(); } } Future cancelPushToTalk() async { _pttButtonHeld = false; liveSessionActive = false; _cancelResponseWatchdog(); final hadActiveTurn = pttStatus == PttStatus.connecting || pttStatus == PttStatus.recording || pttStatus == PttStatus.thinking; try { await _abortMicStreaming(); if (hadActiveTurn && _pttHub != null && _pttHub!.state == HubConnectionState.Connected) { _pttLog('cancelPushToTalk: invoking CancelRecording'); await _pttHub!.invoke("CancelRecording").timeout(const Duration(seconds: 5)); } } catch (e) { _pttLog('cancelPushToTalk: error=$e'); } pttStatus = PttStatus.idle; notifyListeners(); } Future _startMicStreaming() async { final micStatus = await Permission.microphone.request(); if (!micStatus.isGranted) { _pttLog('_startMicStreaming: microphone permission denied'); pttStatus = PttStatus.error; pttErrorMessage = "Microphone permission denied"; notifyListeners(); await cancelPushToTalk(); return; } if (!_pttRecorderOpen) { await _pttRecorder.openRecorder(); _pttRecorderOpen = true; } await _pttRecorder.setSubscriptionDuration(const Duration(milliseconds: 100)); _pttMicStreamController = StreamController(); _pttSendQueue = Future.value(); _pttChunkCount = 0; _pttMicSub = _pttMicStreamController!.stream.listen((chunk) { _pttChunkCount++; _pttLog('mic chunk #$_pttChunkCount captured bytes=${chunk.length}'); _enqueueChunkSend(chunk); if (_pttIsLive) _processSilenceDetection(chunk); }); await _pttRecorder.startRecorder( toStream: _pttMicStreamController!.sink, codec: sound.Codec.pcm16, numChannels: 1, sampleRate: 48000, ); _pttLog('_startMicStreaming: recorder started'); pttStatus = PttStatus.recording; notifyListeners(); if (_pttIsLive) { _startSilenceDetection(); } } /// Resets voice-activity state for a new Live Talk recording. Actual silence /// detection happens per mic chunk in [_processSilenceDetection]. void _startSilenceDetection() { _pttAutoStopTriggered = false; _pttHasDetectedSpeech = false; _pttLastLoudAt = null; _pttNoiseFloor = 0.01; _pttSpeechPeak = 0.05; _pttVoiceActivity = 0.0; } void _stopSilenceDetection() { _pttLastLoudAt = null; _pttHasDetectedSpeech = false; _pttAutoStopTriggered = false; _pttVoiceActivity = 0.0; } /// Live Talk auto-stop: once the user has spoken and then stayed silent for /// [_silenceHoldDuration] (3s), automatically send StopRecording — the button /// is only used to start/stop the whole session, never per-utterance. /// /// Self-calibrating + noise/echo tolerant: /// - Tracks the quiet floor (fast down / slow up) and the speech peak /// (fast up / slow down), and sets a per-chunk threshold 45% of the way /// from floor toward peak. /// - Smooths that per-chunk decision into [_pttVoiceActivity] so a few /// stray room-noise / echo chunks can't masquerade as speech and keep /// resetting the silence countdown. Only *sustained* speech counts. void _processSilenceDetection(Uint8List chunk) { if (_pttAutoStopTriggered) return; final rms = _chunkRms(chunk); final now = DateTime.now(); if (rms < _pttNoiseFloor) { _pttNoiseFloor = rms; } else { _pttNoiseFloor = _pttNoiseFloor * 0.995 + rms * 0.005; } if (rms > _pttSpeechPeak) { _pttSpeechPeak = rms; } else { _pttSpeechPeak = _pttSpeechPeak * 0.995 + rms * 0.005; } final span = _pttSpeechPeak - _pttNoiseFloor; final threshold = max(_pttNoiseFloor + span * 0.45, 0.012); final chunkIsSpeech = rms > threshold && span > 0.01; // Smooth: a single loud chunk only nudges activity, sustained speech climbs. _pttVoiceActivity = _pttVoiceActivity * 0.8 + (chunkIsSpeech ? 1.0 : 0.0) * 0.2; // Hysteresis: need clear sustained energy to count as actively speaking. final activelySpeaking = _pttVoiceActivity > 0.4; _pttLog('silence: rms=${rms.toStringAsFixed(4)} floor=${_pttNoiseFloor.toStringAsFixed(4)} peak=${_pttSpeechPeak.toStringAsFixed(4)} thr=${threshold.toStringAsFixed(4)} act=${_pttVoiceActivity.toStringAsFixed(2)} speaking=$activelySpeaking spoke=$_pttHasDetectedSpeech'); if (activelySpeaking) { _pttLastLoudAt = now; _pttHasDetectedSpeech = true; return; } if (!_pttHasDetectedSpeech) return; final lastLoud = _pttLastLoudAt; if (lastLoud != null && now.difference(lastLoud) >= _silenceHoldDuration) { _pttAutoStopTriggered = true; _pttLog('live: silence after speech, auto-stopping recording'); endPushToTalk().catchError((_) {}); } } /// Root-mean-square amplitude (0..1) of a PCM16 mono little-endian chunk. double _chunkRms(Uint8List chunk) { final sampleCount = chunk.length ~/ 2; if (sampleCount == 0) return 0; final data = ByteData.sublistView(chunk); double sumSquares = 0; for (var i = 0; i < sampleCount; i++) { final sample = data.getInt16(i * 2, Endian.little) / 32768.0; sumSquares += sample * sample; } return sqrt(sumSquares / sampleCount); } Future _stopMicStreamingAndFlush() async { _stopSilenceDetection(); try { if (_pttRecorderOpen) { await _pttRecorder.stopRecorder(); } } catch (e) { _pttLog('_stopMicStreamingAndFlush: stopRecorder error=$e'); } await Future.delayed(const Duration(milliseconds: 50)); await _pttMicSub?.cancel(); _pttMicSub = null; await _pttMicStreamController?.close(); _pttMicStreamController = null; await _pttSendQueue; _pttLog('_stopMicStreamingAndFlush: done, total chunks sent=$_pttChunkCount'); } Future _abortMicStreaming() async { _stopSilenceDetection(); try { if (_pttRecorderOpen) { await _pttRecorder.stopRecorder(); } } catch (e) { _pttLog('_abortMicStreaming: stopRecorder error=$e'); } await _pttMicSub?.cancel(); _pttMicSub = null; await _pttMicStreamController?.close(); _pttMicStreamController = null; _pttSendQueue = Future.value(); } void _enqueueChunkSend(Uint8List chunk) { _pttSendQueue = _pttSendQueue.then((_) => _sendAudioChunk(chunk)); } Future _sendAudioChunk(Uint8List chunk) async { final hub = _pttHub; if (hub == null || hub.state != HubConnectionState.Connected) return; try { await hub.send("SendAudioChunk", args: [base64Encode(chunk)]); } catch (e) { _pttLog('_sendAudioChunk: failed error=$e'); } } void _onRecordingStarted(List? params) async { _pttLog('event RecordingStarted params=$params'); try { if (!_pttButtonHeld) { _pttLog('RecordingStarted received but button already released, cancelling'); try { await _pttHub?.invoke("CancelRecording"); } catch (_) {} pttStatus = PttStatus.idle; notifyListeners(); return; } await _startMicStreaming(); } catch (e) { _pttLog('_onRecordingStarted: failed error=$e'); pttStatus = PttStatus.error; pttErrorMessage = "Failed to start recording"; notifyListeners(); } } void _onRecordingStopped(List? params) { _pttLog('event RecordingStopped params=$params'); } void _onRecordingCancelled(List? params) { _pttLog('event RecordingCancelled params=$params'); pttStatus = PttStatus.idle; notifyListeners(); } void _onTranscriptionReceived(List? params) { _pttLog('event TranscriptionReceived params=$params'); final data = _firstArgAsMap(params); final text = data?['text']?.toString() ?? ''; if (text.isEmpty) return; pttMessages.add(CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.user, content: text, timestamp: DateTime.now())); notifyListeners(); } void _onResponseReceived(List? params) { _pttLog('event ResponseReceived params=$params'); _cancelResponseWatchdog(); final data = _firstArgAsMap(params); final text = data?['voiceAnswer']?.toString() ?? ''; _pttCurrentAssistantMsg = CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.assistant, content: text, timestamp: DateTime.now()); pttMessages.add(_pttCurrentAssistantMsg!); if (liveSessionActive && _pttIsLive) { // Live Talk: let the bot finish speaking its TTS audio, then resume // listening. _maybeRestartLiveListening is debounced so it waits for any // audio clips that arrive right after this text response. pttStatus = PttStatus.playingAudio; notifyListeners(); _maybeRestartLiveListening(); } else { pttStatus = PttStatus.idle; notifyListeners(); } } void _onAudioReceived(List? params) { if (params == null || params.isEmpty || params.first == null) return; try { final bytes = base64Decode(params.first.toString()); _pttLog('event AudioReceived bytes=${bytes.length}'); if (liveSessionActive && _pttIsLive) { // Live Talk: queue and play through the provider so we know when the // bot finished speaking (to resume listening). No widget player. _liveAudioQueue.add(bytes); _playNextLiveClip(); } else { _pttCurrentAssistantMsg?.ttsAudioClips.add(bytes); notifyListeners(); } } catch (e) { _pttLog('_onAudioReceived: failed to decode error=$e'); } } // ----- Live Talk audio playback + auto-resume listening ----- void _playNextLiveClip() { if (_liveIsPlaying || _liveAudioQueue.isEmpty) return; _liveRestartTimer?.cancel(); final clip = _liveAudioQueue.removeAt(0); _liveIsPlaying = true; if (pttStatus != PttStatus.playingAudio) { pttStatus = PttStatus.playingAudio; notifyListeners(); } _livePlayer.play(audioplayers.BytesSource(clip, mimeType: 'audio/wav')).catchError((e) { _pttLog('_playNextLiveClip: play error=$e'); _onLiveClipComplete(); }); } void _onLiveClipComplete() { _liveIsPlaying = false; if (_liveAudioQueue.isNotEmpty) { _playNextLiveClip(); } else { _maybeRestartLiveListening(); } } /// After the bot stops speaking, resume listening. Debounced so trailing /// audio clips (arriving just after) don't cause a premature restart. void _maybeRestartLiveListening() { if (!liveSessionActive || !_pttIsLive) return; _liveRestartTimer?.cancel(); _liveRestartTimer = Timer(const Duration(milliseconds: 600), () { if (!liveSessionActive || !_pttIsLive) return; if (_liveIsPlaying || _liveAudioQueue.isNotEmpty) return; if (pttStatus == PttStatus.recording || pttStatus == PttStatus.connecting) return; _pttLog('live: bot finished speaking, resuming listening'); pttStatus = PttStatus.idle; beginPushToTalk(_pttLanguage ?? '1', isLive: true).catchError((_) {}); }); } void _stopLivePlayback() { _liveRestartTimer?.cancel(); _liveRestartTimer = null; _liveAudioQueue.clear(); _liveIsPlaying = false; _livePlayer.stop().catchError((_) {}); } void _onRecordingError(List? params) { final msg = (params != null && params.isNotEmpty) ? params.first?.toString() : null; _pttLog('event RecordingError message=$msg'); _cancelResponseWatchdog(); pttErrorMessage = msg ?? "Recording error occurred"; pttStatus = PttStatus.error; liveSessionActive = false; _stopLivePlayback(); _abortMicStreaming(); notifyListeners(); } Map? _firstArgAsMap(List? params) { if (params == null || params.isEmpty || params.first == null) return null; final first = params.first; if (first is Map) return Map.from(first); try { final decoded = jsonDecode(jsonEncode(first)); if (decoded is Map) return Map.from(decoded); } catch (_) {} return null; } @override void dispose() { _streamSub?.cancel(); _cancelResponseWatchdog(); _liveRestartTimer?.cancel(); _livePlayer.dispose(); _abortMicStreaming(); if (_pttRecorderOpen) { _pttRecorder.closeRecorder(); } final hub = _pttHub; _pttHub = null; final httpClient = _pttHttpClient; _pttHttpClient = null; hub?.stop().whenComplete(() => httpClient?.closeAll()); super.dispose(); } }