From 7bb4ae420d487badc87e8a1cf509841885b81249 Mon Sep 17 00:00:00 2001 From: WaseemAbbasi22 <50428976+WaseemAbbasi22@users.noreply.github.com> Date: Wed, 21 Oct 2026 11:29:32 +0300 Subject: [PATCH] push to talk completed just have response delays --- lib/controllers/api_routes/urls.dart | 2 +- .../copilot/controller/copilot_provider.dart | 235 +++++++++++------ .../copilot/screens/copilot_chat_page.dart | 8 + .../screens/copilot_live_talk_page.dart | 208 +++++++++++++++ lib/modules/copilot/utils/wav_merge.dart | 95 +++++++ .../widgets/copilot_message_bubble.dart | 14 +- .../copilot/widgets/tts_voice_player.dart | 239 ++++++++++++++++++ 7 files changed, 714 insertions(+), 87 deletions(-) create mode 100644 lib/modules/copilot/screens/copilot_live_talk_page.dart create mode 100644 lib/modules/copilot/utils/wav_merge.dart create mode 100644 lib/modules/copilot/widgets/tts_voice_player.dart diff --git a/lib/controllers/api_routes/urls.dart b/lib/controllers/api_routes/urls.dart index 7fb8e80c..644b8957 100644 --- a/lib/controllers/api_routes/urls.dart +++ b/lib/controllers/api_routes/urls.dart @@ -630,7 +630,7 @@ class URLs { static get predictDevice => "$_baseUrl/Automation/PredictDevice"; static copilotGetConversationByThreadId(String threadId) => "$_baseUrl/Automation/GetConversationByThreadId/$threadId"; static copilotDeleteQueryHistory(String threadId) => "$_baseUrl/Automation/DeleteQueryHistory/$threadId"; - static get copilotHubUrl => "$_baseUrl/VoiceWebSocket"; + static get copilotHubUrl => "$_host/VoiceWebSocket"; } diff --git a/lib/modules/copilot/controller/copilot_provider.dart b/lib/modules/copilot/controller/copilot_provider.dart index 2775d7c1..4e7fc52f 100644 --- a/lib/modules/copilot/controller/copilot_provider.dart +++ b/lib/modules/copilot/controller/copilot_provider.dart @@ -5,7 +5,6 @@ import 'dart:io'; import 'dart:math'; import 'package:signalr_netcore/ihub_protocol.dart'; import 'package:signalr_netcore/signalr_client.dart'; -import 'package:audioplayers/audioplayers.dart' as audioplayers; import 'package:flutter/foundation.dart'; import 'package:flutter_sound/flutter_sound.dart' as sound; import 'package:http/http.dart'; @@ -29,11 +28,17 @@ class CopilotProvider extends ChangeNotifier { StreamSubscription? _streamSub; HubConnection? _pttHub; + // Serializes connect/disconnect so they never overlap when the user navigates + // in and out of the Push-to-Talk screen quickly. + Future _pttOpQueue = Future.value(); PttStatus pttStatus = PttStatus.idle; String? pttErrorMessage; List pttMessages = []; String? pttSessionId; bool _pttButtonHeld = false; + bool _pttIsLive = false; + String? _pttLanguage; + bool liveSessionActive = false; final sound.FlutterSoundRecorder _pttRecorder = sound.FlutterSoundRecorder(); bool _pttRecorderOpen = false; @@ -42,9 +47,15 @@ class CopilotProvider extends ChangeNotifier { Future _pttSendQueue = Future.value(); int _pttChunkCount = 0; - final audioplayers.AudioPlayer _pttAudioPlayer = audioplayers.AudioPlayer(); - final List _pttAudioQueue = []; - bool _pttIsPlayingAudio = false; + // Live Talk voice-activity (silence) detection. + StreamSubscription? _pttProgressSub; + DateTime? _pttLastLoudAt; + DateTime? _pttRecordingStartedAt; + bool _pttAutoStopTriggered = false; + bool _pttHasDetectedSpeech = false; + static const double _silenceDbThreshold = 38.0; + static const Duration _silenceHoldDuration = Duration(milliseconds: 1400); + CopilotMessage? _pttCurrentAssistantMsg; void _pttLog(String message) => log1.log('[PTT] $message'); @@ -342,22 +353,30 @@ class CopilotProvider extends ChangeNotifier { } } - Future connectPushToTalkHub() async { - if (_pttHub != null && _pttHub!.state == HubConnectionState.Connected) { - _pttLog('connectPushToTalkHub: already connected, skipping'); - return; - } - await disconnectPushToTalkHub(); + /// Builds a brand-new push-to-talk connection. Any previous connection is + /// torn down first, so entering the screen always gives a fresh connection. + /// Serialized via [_pttOpQueue] so it never races with a disconnect. + Future connectPushToTalkHub() { + _pttOpQueue = _pttOpQueue.then((_) => _doConnectPushToTalkHub()).catchError((e) { + _pttLog('connectPushToTalkHub: queued error=$e'); + }); + return _pttOpQueue; + } + + Future _doConnectPushToTalkHub() async { + // Always start from a clean slate so every entry is a fresh connection. + await _stopHubAndCleanup(); - // final token = URLs.chatApiKey; final token = ApiManager.instance.user?.token ?? ''; - final userId = ApiManager.instance.user?.id ?? ''; - // final url = "${URLs.copilotHubUrl}?access_token=$token"; - final url = "https://atomsmuat.hmg.com/VoiceWebSocket?access_token=$token"; - // final url = "${URLs.copilotHubUrl}?id=$userId&access_token=$token"; + final url = "${URLs.copilotHubUrl}?access_token=$token"; _pttLog('connectPushToTalkHub: building connection url=$url'); - final httpOp = HttpConnectionOptions(skipNegotiation: false, logMessageContent: true,headers: MessageHeaders()..setHeaderValue('Authorization', 'Bearer $token')); - _pttHub = HubConnectionBuilder().withUrl(url,transportType:HttpTransportType.LongPolling ).withAutomaticReconnect(retryDelays: [2000, 5000, 10000, 20000]).build(); + final httpOp = HttpConnectionOptions( + transport: HttpTransportType.LongPolling, + skipNegotiation: false, + logMessageContent: true, + headers: MessageHeaders()..setHeaderValue('Authorization', 'Bearer $token'), + ); + _pttHub = HubConnectionBuilder().withUrl(url, options: httpOp).withAutomaticReconnect(retryDelays: [2000, 5000, 10000, 20000]).build(); _pttHub!.on("RecordingStarted", _onRecordingStarted); _pttHub!.on("RecordingStopped", _onRecordingStopped); @@ -368,30 +387,50 @@ class CopilotProvider extends ChangeNotifier { _pttHub!.on("RecordingError", _onRecordingError); try { - await _pttHub!.start(); + _pttLog('connectPushToTalkHub: calling start()...'); + await _pttHub!.start()?.timeout(const Duration(seconds: 20), onTimeout: () { + throw TimeoutException('Connection to push-to-talk service timed out'); + }); _pttLog('connectPushToTalkHub: connected, state=${_pttHub!.state}'); - } catch (e) { - _pttLog('connectPushToTalkHub: failed error=$e'); + } catch (e, st) { + _pttLog('connectPushToTalkHub: failed error=$e\n$st'); pttErrorMessage = "Failed to connect to push-to-talk service"; pttStatus = PttStatus.error; - _pttHub = null; + await _stopHubAndCleanup(); notifyListeners(); rethrow; } } - Future disconnectPushToTalkHub() async { + /// Disconnects the push-to-talk connection cleanly. Called when leaving the + /// screen. Serialized via [_pttOpQueue] so it never races with a connect. + Future disconnectPushToTalkHub() { + _pttOpQueue = _pttOpQueue.then((_) => _doDisconnectPushToTalkHub()).catchError((e) { + _pttLog('disconnectPushToTalkHub: queued error=$e'); + }); + return _pttOpQueue; + } + + Future _doDisconnectPushToTalkHub() async { _pttLog('disconnectPushToTalkHub: stopping'); + await _stopHubAndCleanup(); + pttStatus = PttStatus.idle; + } + + /// Stops mic streaming and tears down the hub. Does NOT touch [_pttOpQueue] + /// so it is safe to call from inside queued operations. + Future _stopHubAndCleanup() async { try { await _abortMicStreaming(); if (_pttHub != null) { - await _pttHub!.stop(); + await _pttHub!.stop().timeout(const Duration(seconds: 10), onTimeout: () { + _pttLog('_stopHubAndCleanup: stop() timed out, forcing cleanup'); + }); } } catch (e) { - _pttLog('disconnectPushToTalkHub: error=$e'); + _pttLog('_stopHubAndCleanup: error=$e'); } finally { _pttHub = null; - pttStatus = PttStatus.idle; } } @@ -400,13 +439,17 @@ class CopilotProvider extends ChangeNotifier { pttMessages = []; pttErrorMessage = null; pttStatus = PttStatus.idle; + liveSessionActive = false; + _pttIsLive = false; notifyListeners(); } - Future beginPushToTalk(String language) async { + Future beginPushToTalk(String language, {bool isLive = false}) async { if (pttStatus != PttStatus.idle && pttStatus != PttStatus.error) return; pttSessionId ??= _generateSessionId(); _pttButtonHeld = true; + _pttIsLive = isLive; + _pttLanguage = language; pttErrorMessage = null; pttStatus = PttStatus.connecting; notifyListeners(); @@ -416,26 +459,45 @@ class CopilotProvider extends ChangeNotifier { await connectPushToTalkHub(); } final args = [ - // pttSessionId ?? '', - sessionId ?? '', - ApiManager.instance.user?.id ?? '', - (ApiManager.instance.assetGroup?.id ?? '').toString(), - // language, - "1", - false, + { + "sessionId": pttSessionId ?? '', + "userId": ApiManager.instance.user?.id ?? '', + "assetGroupId": (ApiManager.instance.assetGroup?.id ?? '').toString(), + "languageId": "1", + "isLiveSession": isLive, + } ]; _pttLog('beginPushToTalk: invoking StartRecording args=$args'); await _pttHub!.invoke("StartRecording", args: args); _pttLog('beginPushToTalk: StartRecording invoked, waiting for RecordingStarted event'); - } catch (e) { - _pttLog('beginPushToTalk: failed error=$e'); + } catch (e, st) { + _pttLog('beginPushToTalk: failed error=$e\n$st'); pttStatus = PttStatus.error; pttErrorMessage = "Failed to start recording"; _pttButtonHeld = false; + liveSessionActive = false; notifyListeners(); } } + /// Starts a continuous "Live Talk" session: records, waits for a response, + /// then automatically starts listening again until [stopLiveTalk] is called. + Future startLiveTalk(String language) async { + if (pttStatus != PttStatus.idle && pttStatus != PttStatus.error) return; + liveSessionActive = true; + await beginPushToTalk(language, isLive: true); + } + + /// Stops the current Live Talk session (does not auto-restart listening again). + Future stopLiveTalk() async { + liveSessionActive = false; + if (pttStatus == PttStatus.recording) { + await endPushToTalk(); + } else { + await cancelPushToTalk(); + } + } + Future endPushToTalk() async { _pttButtonHeld = false; if (pttStatus != PttStatus.recording) { @@ -443,13 +505,16 @@ class CopilotProvider extends ChangeNotifier { return; } + pttStatus = PttStatus.thinking; + notifyListeners(); + _pttLog('endPushToTalk: stopping mic and flushing $_pttChunkCount chunks sent so far'); try { await _stopMicStreamingAndFlush(); - pttStatus = PttStatus.thinking; - notifyListeners(); - _pttLog('endPushToTalk: invoking StopRecording'); - await _pttHub?.invoke("StopRecording"); + if (_pttHub != null && _pttHub!.state == HubConnectionState.Connected) { + _pttLog('endPushToTalk: invoking StopRecording'); + await _pttHub!.invoke("StopRecording"); + } } catch (e) { _pttLog('endPushToTalk: failed error=$e'); pttStatus = PttStatus.error; @@ -460,10 +525,13 @@ class CopilotProvider extends ChangeNotifier { Future cancelPushToTalk() async { _pttButtonHeld = false; - _pttLog('cancelPushToTalk: invoking CancelRecording'); + liveSessionActive = false; try { await _abortMicStreaming(); - await _pttHub?.invoke("CancelRecording"); + if (_pttHub != null && _pttHub!.state == HubConnectionState.Connected) { + _pttLog('cancelPushToTalk: invoking CancelRecording'); + await _pttHub!.invoke("CancelRecording"); + } } catch (e) { _pttLog('cancelPushToTalk: error=$e'); } @@ -508,9 +576,51 @@ class CopilotProvider extends ChangeNotifier { _pttLog('_startMicStreaming: recorder started'); pttStatus = PttStatus.recording; notifyListeners(); + + if (_pttIsLive) { + _startSilenceDetection(); + } + } + + /// Monitors mic decibel levels while in Live Talk mode and automatically + /// triggers [endPushToTalk] once the user has been silent for + /// [_silenceHoldDuration] AFTER having spoken, so the user doesn't need to + /// tap anything. Won't trigger on the initial silence before the user + /// starts speaking. + void _startSilenceDetection() { + _pttAutoStopTriggered = false; + _pttHasDetectedSpeech = false; + _pttRecordingStartedAt = DateTime.now(); + _pttLastLoudAt = null; + _pttProgressSub?.cancel(); + _pttProgressSub = _pttRecorder.onProgress?.listen((disposition) { + final db = disposition.decibels ?? 0; + final now = DateTime.now(); + if (db > _silenceDbThreshold) { + _pttLastLoudAt = now; + _pttHasDetectedSpeech = true; + return; + } + if (_pttAutoStopTriggered || !_pttHasDetectedSpeech) return; + final lastLoud = _pttLastLoudAt; + if (lastLoud != null && now.difference(lastLoud) >= _silenceHoldDuration) { + _pttAutoStopTriggered = true; + _pttLog('_startSilenceDetection: silence detected after speech, auto-stopping recording (live talk)'); + endPushToTalk().catchError((_) {}); + } + }); + } + + void _stopSilenceDetection() { + _pttProgressSub?.cancel(); + _pttProgressSub = null; + _pttLastLoudAt = null; + _pttRecordingStartedAt = null; + _pttHasDetectedSpeech = false; } Future _stopMicStreamingAndFlush() async { + _stopSilenceDetection(); try { if (_pttRecorderOpen) { await _pttRecorder.stopRecorder(); @@ -528,6 +638,7 @@ class CopilotProvider extends ChangeNotifier { } Future _abortMicStreaming() async { + _stopSilenceDetection(); try { if (_pttRecorderOpen) { await _pttRecorder.stopRecorder(); @@ -548,7 +659,7 @@ class CopilotProvider extends ChangeNotifier { Future _sendAudioChunk(Uint8List chunk) async { try { - await _pttHub?.invoke("SendAudioChunk", args: [base64Encode(chunk)]); + await _pttHub?.send("SendAudioChunk", args: [base64Encode(chunk)]); } catch (e) { _pttLog('_sendAudioChunk: failed error=$e'); } @@ -600,7 +711,14 @@ class CopilotProvider extends ChangeNotifier { final text = data?['voiceAnswer']?.toString() ?? ''; _pttCurrentAssistantMsg = CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.assistant, content: text, timestamp: DateTime.now()); pttMessages.add(_pttCurrentAssistantMsg!); + pttStatus = PttStatus.idle; notifyListeners(); + + if (liveSessionActive && _pttIsLive) { + final lang = _pttLanguage ?? '1'; + _pttLog('ResponseReceived: live session active, auto-restarting listening'); + Future.microtask(() => beginPushToTalk(lang, isLive: true).catchError((_) {})); + } } void _onAudioReceived(List? params) { @@ -609,9 +727,7 @@ class CopilotProvider extends ChangeNotifier { final bytes = base64Decode(params.first.toString()); _pttLog('event AudioReceived bytes=${bytes.length}'); _pttCurrentAssistantMsg?.ttsAudioClips.add(bytes); - _pttAudioQueue.add(bytes); notifyListeners(); - _playNextTtsClipIfIdle(); } catch (e) { _pttLog('_onAudioReceived: failed to decode error=$e'); } @@ -622,6 +738,7 @@ class CopilotProvider extends ChangeNotifier { _pttLog('event RecordingError message=$msg'); pttErrorMessage = msg ?? "Recording error occurred"; pttStatus = PttStatus.error; + liveSessionActive = false; _abortMicStreaming(); notifyListeners(); } @@ -637,35 +754,6 @@ class CopilotProvider extends ChangeNotifier { return null; } - Future _playNextTtsClipIfIdle() async { - if (_pttIsPlayingAudio) return; - if (_pttAudioQueue.isEmpty) return; - _pttIsPlayingAudio = true; - pttStatus = PttStatus.playingAudio; - notifyListeners(); - - while (_pttAudioQueue.isNotEmpty) { - final clip = _pttAudioQueue.removeAt(0); - final completer = Completer(); - late StreamSubscription sub; - sub = _pttAudioPlayer.onPlayerComplete.listen((_) { - sub.cancel(); - if (!completer.isCompleted) completer.complete(); - }); - try { - await _pttAudioPlayer.play(audioplayers.BytesSource(clip, mimeType: 'audio/wav')); - await completer.future; - } catch (_) { - sub.cancel(); - } - } - - _pttIsPlayingAudio = false; - pttStatus = PttStatus.idle; - notifyListeners(); - } - - @override void dispose() { _streamSub?.cancel(); @@ -673,7 +761,6 @@ class CopilotProvider extends ChangeNotifier { if (_pttRecorderOpen) { _pttRecorder.closeRecorder(); } - _pttAudioPlayer.dispose(); _pttHub?.stop(); super.dispose(); } diff --git a/lib/modules/copilot/screens/copilot_chat_page.dart b/lib/modules/copilot/screens/copilot_chat_page.dart index 3c00448b..a11583db 100644 --- a/lib/modules/copilot/screens/copilot_chat_page.dart +++ b/lib/modules/copilot/screens/copilot_chat_page.dart @@ -11,6 +11,7 @@ import 'package:test_sa/extensions/string_extensions.dart'; import 'package:test_sa/extensions/text_extensions.dart'; import 'package:test_sa/extensions/widget_extensions.dart'; import 'package:test_sa/modules/copilot/screens/copilot_history_page.dart'; +import 'package:test_sa/modules/copilot/screens/copilot_live_talk_page.dart'; import 'package:test_sa/modules/copilot/screens/copilot_push_to_talk_page.dart'; import 'package:test_sa/new_views/app_style/app_color.dart'; import 'package:test_sa/new_views/common_widgets/default_app_bar.dart'; @@ -127,6 +128,13 @@ class _CopilotChatPageState extends State { appBar: DefaultAppBar( title: "ATOMS – AI Assistant", actions: [ + IconButton( + onPressed: () => Navigator.push(context, MaterialPageRoute(builder: (_) => const CopilotLiveTalkPage())), + style: const ButtonStyle(tapTargetSize: MaterialTapTargetSize.shrinkWrap), + icon: Icon(Icons.graphic_eq_rounded, color: AppColor.icon2Color(context)), + constraints: const BoxConstraints(), + tooltip: "Live Talk", + ), IconButton( onPressed: () => Navigator.push(context, MaterialPageRoute(builder: (_) => const CopilotPushToTalkPage())), style: const ButtonStyle(tapTargetSize: MaterialTapTargetSize.shrinkWrap), diff --git a/lib/modules/copilot/screens/copilot_live_talk_page.dart b/lib/modules/copilot/screens/copilot_live_talk_page.dart new file mode 100644 index 00000000..5d7538b8 --- /dev/null +++ b/lib/modules/copilot/screens/copilot_live_talk_page.dart @@ -0,0 +1,208 @@ +import 'package:flutter/material.dart'; +import 'package:provider/provider.dart'; +import 'package:test_sa/controllers/providers/api/user_provider.dart'; +import 'package:test_sa/extensions/context_extension.dart'; +import 'package:test_sa/extensions/int_extensions.dart'; +import 'package:test_sa/extensions/text_extensions.dart'; +import 'package:test_sa/extensions/widget_extensions.dart'; +import 'package:test_sa/new_views/app_style/app_color.dart'; +import 'package:test_sa/new_views/common_widgets/default_app_bar.dart'; +import '../controller/copilot_provider.dart'; +import '../widgets/copilot_message_bubble.dart'; + +/// Live Talk realtime voice screen for Copilot. +/// Unlike Push-to-Talk (press & hold), this is a continuous conversation mode: +/// tap once to start, the app keeps listening/responding automatically until +/// the user taps again to stop. +class CopilotLiveTalkPage extends StatefulWidget { + const CopilotLiveTalkPage({Key? key}) : super(key: key); + + @override + State createState() => _CopilotLiveTalkPageState(); +} + +class _CopilotLiveTalkPageState extends State { + final ScrollController _scrollController = ScrollController(); + CopilotProvider? _provider; + + @override + void didChangeDependencies() { + super.didChangeDependencies(); + _provider = context.read(); + } + + @override + void initState() { + super.initState(); + WidgetsBinding.instance.addPostFrameCallback((_) { + final provider = context.read(); + provider.startNewPttConversation(); + provider.connectPushToTalkHub().catchError((_) {}); + }); + } + + @override + void dispose() { + _scrollController.dispose(); + _provider?.stopLiveTalk().catchError((_) {}); + _provider?.disconnectPushToTalkHub().catchError((_) {}); + super.dispose(); + } + + void _scrollToBottom() { + WidgetsBinding.instance.addPostFrameCallback((_) { + if (_scrollController.hasClients) { + _scrollController.animateTo( + _scrollController.position.maxScrollExtent, + duration: const Duration(milliseconds: 250), + curve: Curves.easeOut, + ); + } + }); + } + + String _statusLabel(PttStatus status, bool isLiveActive) { + switch (status) { + case PttStatus.idle: + return isLiveActive ? "Getting ready…" : "Tap to start Live Talk"; + case PttStatus.connecting: + return "Connecting…"; + case PttStatus.recording: + return "Listening… (pause to send)"; + case PttStatus.thinking: + return "Thinking…"; + case PttStatus.playingAudio: + return "Speaking…"; + case PttStatus.error: + return "Something went wrong"; + } + } + + void _onMainButtonTap(CopilotProvider provider) { + final status = provider.pttStatus; + final isBusy = status == PttStatus.connecting || status == PttStatus.thinking; + if (isBusy) return; + + if (provider.liveSessionActive) { + provider.stopLiveTalk().catchError((_) {}); + } else { + provider.startLiveTalk(context.settingProvider.language!).catchError((_) {}); + } + } + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: const DefaultAppBar(title: "Live Talk"), + body: Column( + children: [ + Expanded( + child: Consumer( + builder: (context, provider, _) { + if (provider.pttMessages.isEmpty) { + return Column( + crossAxisAlignment: CrossAxisAlignment.center, + spacing: 24, + mainAxisSize: MainAxisSize.min, + children: [ + "atoms_logo".toSvgAsset(width: 54), + Text( + "Tap the mic to start a live conversation", + overflow: TextOverflow.ellipsis, + maxLines: 1, + style: AppTextStyles.heading6.copyWith(color: AppColor.lightTextColor(context), fontWeight: FontWeight.w500), + ), + ], + ).center; + } + + _scrollToBottom(); + final messages = provider.pttMessages; + return ListView.builder( + controller: _scrollController, + padding: const EdgeInsets.all(16), + itemCount: messages.length, + itemBuilder: (context, index) { + final currentMessage = messages[index]; + final bool showHeader = index == 0 || currentMessage.role != messages[index - 1].role; + return CopilotMessageBubble( + message: currentMessage, + showHeader: showHeader, + userAvatar: Provider.of(context, listen: false).user?.profilePhotoName, + ); + }, + ); + }, + ), + ), + Consumer( + builder: (context, provider, _) { + if (provider.pttErrorMessage == null) return const SizedBox(); + return Container( + width: double.infinity, + color: Colors.red.withValues(alpha: 0.1), + padding: const EdgeInsets.all(8), + child: Text(provider.pttErrorMessage!, style: const TextStyle(color: Colors.red)), + ); + }, + ), + SafeArea( + child: Consumer( + builder: (context, provider, _) { + final status = provider.pttStatus; + final isActive = provider.liveSessionActive; + final isRecording = status == PttStatus.recording; + final isBusy = status == PttStatus.connecting || status == PttStatus.thinking; + + return Column( + mainAxisSize: MainAxisSize.min, + children: [ + Text( + _statusLabel(status, isActive), + style: AppTextStyles.bodyText2.copyWith(color: AppColor.lightTextColor(context)), + ), + 16.height, + GestureDetector( + onTap: isBusy ? null : () => _onMainButtonTap(provider), + child: AnimatedContainer( + duration: const Duration(milliseconds: 150), + height: isRecording || isActive ? 96 : 80, + width: isRecording || isActive ? 96 : 80, + decoration: BoxDecoration( + shape: BoxShape.circle, + color: isRecording + ? Colors.red + : isActive + ? AppColor.primary10.withValues(alpha: 0.8) + : AppColor.primary10, + ), + child: isBusy + ? const Padding( + padding: EdgeInsets.all(24), + child: CircularProgressIndicator(color: Colors.white, strokeWidth: 2.5), + ) + : Icon( + isActive ? Icons.stop_rounded : Icons.graphic_eq_rounded, + color: Colors.white, + size: isRecording || isActive ? 44 : 36, + ), + ), + ), + 8.height, + if (isActive) + Text( + "Tap to stop", + style: AppTextStyles.bodyText2.copyWith(color: AppColor.lightTextColor(context)), + ), + 24.height, + ], + ).paddingOnly(top: 8, bottom: 8); + }, + ), + ), + ], + ), + ); + } +} + diff --git a/lib/modules/copilot/utils/wav_merge.dart b/lib/modules/copilot/utils/wav_merge.dart new file mode 100644 index 00000000..27aa11f9 --- /dev/null +++ b/lib/modules/copilot/utils/wav_merge.dart @@ -0,0 +1,95 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +/// Parsed RIFF/WAVE chunks we care about from a single clip. +class _WavChunks { + final Uint8List? fmt; + final Uint8List data; + + const _WavChunks({required this.fmt, required this.data}); +} + +_WavChunks _parseWav(Uint8List clip) { + Uint8List? fmtChunk; + final dataChunks = BytesBuilder(); + if (clip.length < 12) return _WavChunks(fmt: fmtChunk, data: dataChunks.toBytes()); + + final data = ByteData.sublistView(clip); + int offset = 12; // skip "RIFF" size "WAVE" + while (offset + 8 <= clip.length) { + final chunkId = ascii.decode(clip.sublist(offset, offset + 4), allowInvalid: true); + final chunkSize = data.getUint32(offset + 4, Endian.little); + final chunkStart = offset + 8; + final chunkEnd = (chunkStart + chunkSize).clamp(0, clip.length); + + if (chunkId == 'fmt ') { + fmtChunk ??= clip.sublist(chunkStart, chunkEnd); + } else if (chunkId == 'data') { + dataChunks.add(clip.sublist(chunkStart, chunkEnd)); + } + + offset = chunkStart + chunkSize + (chunkSize.isOdd ? 1 : 0); + } + + return _WavChunks(fmt: fmtChunk, data: dataChunks.toBytes()); +} + +/// Merges multiple small WAV clips (as received one-per-sentence from the +/// Push-to-Talk hub's `AudioReceived` event) into a single playable WAV byte +/// buffer. +/// +/// Assumes all clips share the same PCM format (sample rate/channels/bit +/// depth), which holds true since they all come from the same TTS voice +/// within a single response. +Uint8List mergeWavClips(List clips) { + if (clips.isEmpty) return Uint8List(0); + if (clips.length == 1) return clips.first; + + Uint8List? fmtChunk; + final pcmData = BytesBuilder(); + + for (final clip in clips) { + final parsed = _parseWav(clip); + fmtChunk ??= parsed.fmt; + pcmData.add(parsed.data); + } + + fmtChunk ??= Uint8List(16); + final pcmBytes = pcmData.toBytes(); + final fmtSize = fmtChunk.length; + final dataSize = pcmBytes.length; + final riffSize = 4 + (8 + fmtSize) + (8 + dataSize); + + final header = BytesBuilder(); + header.add(ascii.encode('RIFF')); + header.add(_uint32le(riffSize)); + header.add(ascii.encode('WAVE')); + header.add(ascii.encode('fmt ')); + header.add(_uint32le(fmtSize)); + header.add(fmtChunk); + header.add(ascii.encode('data')); + header.add(_uint32le(dataSize)); + + return Uint8List.fromList([...header.toBytes(), ...pcmBytes]); +} + +/// Computes a WAV clip's playback duration directly from its header +/// (dataSize / byteRate), without needing to load it into an audio engine. +Duration wavDuration(Uint8List clip) { + final parsed = _parseWav(clip); + final fmt = parsed.fmt; + if (fmt == null || fmt.length < 12 || parsed.data.isEmpty) return Duration.zero; + + final byteRate = ByteData.sublistView(fmt).getUint32(8, Endian.little); + if (byteRate == 0) return Duration.zero; + + final seconds = parsed.data.length / byteRate; + return Duration(milliseconds: (seconds * 1000).round()); +} + +Uint8List _uint32le(int value) { + final b = ByteData(4); + b.setUint32(0, value, Endian.little); + return b.buffer.asUint8List(); +} + diff --git a/lib/modules/copilot/widgets/copilot_message_bubble.dart b/lib/modules/copilot/widgets/copilot_message_bubble.dart index 0d1d30a8..aa1f544f 100644 --- a/lib/modules/copilot/widgets/copilot_message_bubble.dart +++ b/lib/modules/copilot/widgets/copilot_message_bubble.dart @@ -1,5 +1,4 @@ import 'dart:convert'; -import 'dart:io'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter_markdown_plus/flutter_markdown_plus.dart'; @@ -13,6 +12,7 @@ import 'package:test_sa/views/widgets/item_views/info_text_widget.dart'; import 'package:test_sa/views/widgets/sound/sound_player.dart'; import 'package:url_launcher/url_launcher.dart'; import '../model/copilot_message_model.dart'; +import 'tts_voice_player.dart'; class CopilotMessageBubble extends StatelessWidget { final String? userAvatar; @@ -206,17 +206,7 @@ class CopilotMessageBubble extends StatelessWidget { ], if (message.ttsAudioClips.isNotEmpty) ...[ 8.height, - Row( - mainAxisSize: MainAxisSize.min, - children: [ - Icon(Icons.volume_up_rounded, size: 16, color: AppColor.icon2Color(context)), - 4.width, - Text( - "Voice reply", - style: AppTextStyles.bodyText2.copyWith(color: AppColor.textColor(context)), - ), - ], - ), + TtsVoicePlayer(key: ValueKey('${message.id}_audio'), clips: message.ttsAudioClips), ], Align( alignment: Alignment.centerRight, diff --git a/lib/modules/copilot/widgets/tts_voice_player.dart b/lib/modules/copilot/widgets/tts_voice_player.dart new file mode 100644 index 00000000..6c533cf5 --- /dev/null +++ b/lib/modules/copilot/widgets/tts_voice_player.dart @@ -0,0 +1,239 @@ +import 'dart:typed_data'; +import 'package:audioplayers/audioplayers.dart'; +import 'package:flutter/material.dart'; +import 'package:test_sa/extensions/int_extensions.dart'; +import 'package:test_sa/extensions/text_extensions.dart'; +import 'package:test_sa/extensions/widget_extensions.dart'; +import 'package:test_sa/new_views/app_style/app_color.dart'; +import '../utils/wav_merge.dart'; + +/// A self-contained voice-message style player for Push-to-Talk responses, +/// with play/pause, seek, playback speed, and mute controls. +class TtsVoicePlayer extends StatefulWidget { + final List clips; + + const TtsVoicePlayer({Key? key, required this.clips}) : super(key: key); + + @override + State createState() => _TtsVoicePlayerState(); +} + +class _TtsVoicePlayerState extends State { + static const List _speeds = [1.0, 1.25, 1.5, 2.0, 0.75]; + + final AudioPlayer _player = AudioPlayer(); + int _index = -1; + Duration _clipPosition = Duration.zero; + bool _playing = false; + bool _userPaused = false; + bool _muted = false; + double _speed = 1.0; + bool _sliderMoving = false; + double? _previewTotalMs; + bool _finished = false; + + List get _clipDurations => widget.clips.map(wavDuration).toList(); + + Duration get _totalDuration => _clipDurations.fold(Duration.zero, (a, b) => a + b); + + Duration get _elapsedBeforeCurrent { + if (_index <= 0) return Duration.zero; + return _clipDurations.take(_index).fold(Duration.zero, (a, b) => a + b); + } + + Duration get _totalElapsed => _finished ? Duration.zero : _elapsedBeforeCurrent + _clipPosition; + + @override + void initState() { + super.initState(); + _player.setReleaseMode(ReleaseMode.stop); + _player.onPositionChanged.listen((p) { + if (!_sliderMoving && mounted) setState(() => _clipPosition = p); + }); + _player.onPlayerStateChanged.listen((state) { + if (mounted) setState(() => _playing = state == PlayerState.playing); + }); + _player.onPlayerComplete.listen((_) => _onClipComplete()); + _autoAdvanceIfNeeded(); + } + + @override + void didUpdateWidget(covariant TtsVoicePlayer oldWidget) { + super.didUpdateWidget(oldWidget); + if (widget.clips.length != oldWidget.clips.length) { + _autoAdvanceIfNeeded(); + } + } + + @override + void dispose() { + _player.dispose(); + super.dispose(); + } + + void _autoAdvanceIfNeeded() { + if (_userPaused || widget.clips.isEmpty) return; + if (_index == -1) { + _playClipAt(0); + } else if (!_playing && _index == widget.clips.length - 2) { + // We were waiting for more clips after finishing the previous-to-last one. + _playClipAt(_index + 1); + } + } + + void _onClipComplete() { + if (_userPaused) return; + if (_index + 1 < widget.clips.length) { + _playClipAt(_index + 1); + } else { + setState(() { + _playing = false; + _finished = true; + _clipPosition = Duration.zero; + }); + } + } + + Future _playClipAt(int i) async { + if (i < 0 || i >= widget.clips.length) return; + setState(() { + _index = i; + _clipPosition = Duration.zero; + _finished = false; + }); + await _player.play(BytesSource(widget.clips[i], mimeType: 'audio/wav')); + await _player.setPlaybackRate(_speed); + if (_muted) await _player.setVolume(0); + } + + Future _togglePlayPause() async { + if (_playing) { + _userPaused = true; + await _player.pause(); + return; + } + _userPaused = false; + if (_index == -1 || _finished) { + await _playClipAt(0); + return; + } + await _player.resume(); + } + + Future _toggleMute() async { + setState(() => _muted = !_muted); + await _player.setVolume(_muted ? 0 : 1); + } + + Future _cycleSpeed() async { + final idx = _speeds.indexOf(_speed); + final next = _speeds[(idx + 1) % _speeds.length]; + setState(() => _speed = next); + await _player.setPlaybackRate(next); + } + + Future _seekToTotal(Duration target) async { + final durations = _clipDurations; + Duration acc = Duration.zero; + for (var i = 0; i < durations.length; i++) { + final clipEnd = acc + durations[i]; + final isLast = i == durations.length - 1; + if (target < clipEnd || isLast) { + final offset = target - acc; + _userPaused = false; + if (i != _index || _finished) { + await _playClipAt(i); + } + await _player.seek(offset.isNegative ? Duration.zero : offset); + return; + } + acc = clipEnd; + } + } + + String _format(Duration? d) { + final dur = d ?? Duration.zero; + final m = dur.inMinutes.remainder(60).toString().padLeft(2, '0'); + final s = dur.inSeconds.remainder(60).toString().padLeft(2, '0'); + return "$m:$s"; + } + + String get _speedLabel => _speed == _speed.roundToDouble() ? "${_speed.toInt()}x" : "${_speed}x"; + + IconData get _playIcon { + if (_playing) return Icons.pause_rounded; + if (_finished) return Icons.replay_rounded; + return Icons.play_arrow_rounded; + } + + @override + Widget build(BuildContext context) { + final totalMs = _totalDuration.inMilliseconds.toDouble(); + final elapsedMs = (_previewTotalMs ?? _totalElapsed.inMilliseconds.toDouble()).clamp(0, totalMs == 0 ? 1 : totalMs); + + return Container( + padding: const EdgeInsets.symmetric(horizontal: 4, vertical: 2), + decoration: BoxDecoration( + color: AppColor.background(context), + borderRadius: BorderRadius.circular(24), + border: Border.all(color: AppColor.border2Color(context)), + ), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + IconButton( + onPressed: _togglePlayPause, + style: const ButtonStyle(tapTargetSize: MaterialTapTargetSize.shrinkWrap), + constraints: const BoxConstraints(), + icon: Icon(_playIcon, color: AppColor.primary10, size: 26), + ), + SliderTheme( + data: SliderTheme.of(context).copyWith( + trackHeight: 2.0, + thumbShape: const RoundSliderThumbShape(enabledThumbRadius: 5.0), + overlayShape: SliderComponentShape.noOverlay, + ), + child: Slider( + value: elapsedMs.toDouble(), + min: 0, + max: totalMs == 0 ? 1 : totalMs, + activeColor: AppColor.primary10, + inactiveColor: AppColor.border2Color(context), + onChangeStart: (_) => _sliderMoving = true, + onChanged: (value) => setState(() => _previewTotalMs = value), + onChangeEnd: (value) async { + _sliderMoving = false; + _previewTotalMs = null; + await _seekToTotal(Duration(milliseconds: value.round())); + }, + ), + ).expanded, + Text( + "${_format(Duration(milliseconds: elapsedMs.round()))}/${_format(_totalDuration)}", + style: AppTextStyles.tinyFont.copyWith(color: AppColor.textStyleColor(context)), + ), + 4.width, + IconButton( + onPressed: _cycleSpeed, + style: const ButtonStyle(tapTargetSize: MaterialTapTargetSize.shrinkWrap), + constraints: const BoxConstraints(), + tooltip: "Playback speed", + icon: Text( + _speedLabel, + style: AppTextStyles.tinyFont.copyWith(color: AppColor.textStyleColor(context), fontWeight: FontWeight.w600), + ), + ), + IconButton( + onPressed: _toggleMute, + style: const ButtonStyle(tapTargetSize: MaterialTapTargetSize.shrinkWrap), + constraints: const BoxConstraints(), + tooltip: _muted ? "Unmute" : "Mute", + icon: Icon(_muted ? Icons.volume_off_rounded : Icons.volume_up_rounded, size: 18, color: AppColor.icon2Color(context)), + ), + ], + ), + ); + } +} + +