diff --git a/lib/controllers/api_routes/api_manager.dart b/lib/controllers/api_routes/api_manager.dart index 4d5ac9bc..2aa3a88d 100644 --- a/lib/controllers/api_routes/api_manager.dart +++ b/lib/controllers/api_routes/api_manager.dart @@ -271,6 +271,59 @@ class ApiManager { return streamedResponse; } + /// POST FormData request with a file attachment that waits for and returns the + /// full [http.Response] (use this when the endpoint returns a plain/JSON body + /// instead of a stream, e.g. audio transcription). + Future postFormDataWithFile( + String url, { + Map? headers, + Map fields = const {}, + required String fileFieldName, + required File file, + String? filename, + bool showToast = true, + }) async { + Map formHeaders = { + 'X-Timezone-Offset': DateTime.now().timeZoneOffset.toString().split(".").first, + if (user != null) 'Authorization': 'Bearer ${user!.token}', + if (assetGroup != null) 'AssetGroup': assetGroup!.id.toString(), + }; + + headers ??= {}; + headers.addAll(formHeaders); + + Uri url0 = Uri.parse(url); + if (!kReleaseMode) { + print("URL:$url0"); + print("Headers:$headers"); + print("FormData Fields:$fields, File:${file.path}"); + } + + var request = http.MultipartRequest('POST', url0); + request.fields.addAll(fields); + request.headers.addAll(headers); + request.files.add(await http.MultipartFile.fromPath(fileFieldName, file.path, filename: filename)); + + http.StreamedResponse streamedResponse = await _httpClient.send(request); + http.Response response = await http.Response.fromStream(streamedResponse); + + try { + if (response.statusCode == 401) { + showLoginDialog(); + } else { + final responseBody = jsonDecode(response.body); + if (responseBody is Map) { + final message = responseBody["message"]; + if (message != null && message.toString().isNotEmpty && showToast) { + Fluttertoast.showToast(msg: message, toastLength: Toast.LENGTH_LONG); + } + } + } + } catch (ex) {} + + return response; + } + Future delete(String url, {Map? headers, required Map body}) async { headers ??= {}; diff --git a/lib/controllers/api_routes/urls.dart b/lib/controllers/api_routes/urls.dart index 5b529716..7fb8e80c 100644 --- a/lib/controllers/api_routes/urls.dart +++ b/lib/controllers/api_routes/urls.dart @@ -630,4 +630,7 @@ class URLs { static get predictDevice => "$_baseUrl/Automation/PredictDevice"; static copilotGetConversationByThreadId(String threadId) => "$_baseUrl/Automation/GetConversationByThreadId/$threadId"; static copilotDeleteQueryHistory(String threadId) => "$_baseUrl/Automation/DeleteQueryHistory/$threadId"; + static get copilotHubUrl => "$_baseUrl/VoiceWebSocket"; + + } diff --git a/lib/modules/copilot/controller/copilot_provider.dart b/lib/modules/copilot/controller/copilot_provider.dart index af582861..2775d7c1 100644 --- a/lib/modules/copilot/controller/copilot_provider.dart +++ b/lib/modules/copilot/controller/copilot_provider.dart @@ -3,15 +3,20 @@ import 'dart:convert'; import 'dart:developer' as log1; import 'dart:io'; import 'dart:math'; -// import 'dart:math'; - +import 'package:signalr_netcore/ihub_protocol.dart'; +import 'package:signalr_netcore/signalr_client.dart'; +import 'package:audioplayers/audioplayers.dart' as audioplayers; import 'package:flutter/foundation.dart'; +import 'package:flutter_sound/flutter_sound.dart' as sound; import 'package:http/http.dart'; +import 'package:permission_handler/permission_handler.dart'; import 'package:test_sa/controllers/api_routes/api_manager.dart'; import 'package:test_sa/controllers/api_routes/urls.dart'; import '../model/copilot_message_model.dart'; +enum PttStatus { idle, connecting, recording, thinking, playingAudio, error } + class CopilotProvider extends ChangeNotifier { List history = []; bool historyLoading = false; @@ -23,6 +28,27 @@ class CopilotProvider extends ChangeNotifier { String? errorMessage; StreamSubscription? _streamSub; + HubConnection? _pttHub; + PttStatus pttStatus = PttStatus.idle; + String? pttErrorMessage; + List pttMessages = []; + String? pttSessionId; + bool _pttButtonHeld = false; + + final sound.FlutterSoundRecorder _pttRecorder = sound.FlutterSoundRecorder(); + bool _pttRecorderOpen = false; + StreamController? _pttMicStreamController; + StreamSubscription? _pttMicSub; + Future _pttSendQueue = Future.value(); + int _pttChunkCount = 0; + + final audioplayers.AudioPlayer _pttAudioPlayer = audioplayers.AudioPlayer(); + final List _pttAudioQueue = []; + bool _pttIsPlayingAudio = false; + CopilotMessage? _pttCurrentAssistantMsg; + + void _pttLog(String message) => log1.log('[PTT] $message'); + Future loadHistory() async { historyLoading = true; errorMessage = null; @@ -30,7 +56,7 @@ class CopilotProvider extends ChangeNotifier { try { final res = await ApiManager.instance.get( URLs.copilotGetQueryHistory, - enableToastMessage: false, // "message" is just an informational success string here + enableToastMessage: false, ); final decoded = jsonDecode(res.body); if (decoded is Map && decoded['isSuccess'] == true) { @@ -58,7 +84,7 @@ class CopilotProvider extends ChangeNotifier { history.removeWhere((h) => h.threadId == threadId); notifyListeners(); } else { - history = prevList; // rollback on failure + history = prevList; errorMessage = (decoded is Map ? decoded['message'] : null) ?? "Failed to delete"; notifyListeners(); } @@ -105,7 +131,7 @@ class CopilotProvider extends ChangeNotifier { } Future sendText(String query, String language) async { - if (isStreaming || query.trim().isEmpty) return; // prevent invalid duplicate sends + if (isStreaming || query.trim().isEmpty) return; sessionId ??= _generateSessionId(); messages.add(CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.user, content: query, timestamp: DateTime.now())); @@ -166,13 +192,8 @@ class CopilotProvider extends ChangeNotifier { isTranscribing = true; errorMessage = null; notifyListeners(); - log1.log('i am here..'); final exists = await audioFile.exists(); final sizeBytes = exists ? await audioFile.length() : 0; - log1.log( - '[CopilotProvider.transcribeAudio] url=${URLs.copilotAudioTranscription} ' - 'langId=$language path=${audioFile.path} exists=$exists sizeBytes=$sizeBytes', - ); if (!exists || sizeBytes == 0) { errorMessage = "Recording file is empty or missing"; isTranscribing = false; @@ -192,19 +213,13 @@ class CopilotProvider extends ChangeNotifier { ], ); - log1.log( - '[CopilotProvider.transcribeAudio] statusCode=${response.statusCode} ' - 'bodyLength=${response.body.length} body=${response.body}', - ); - if (response.statusCode < 200 || response.statusCode >= 300) { errorMessage = "Failed to transcribe audio"; return null; } return _extractTranscriptionText(response.body); - } catch (e, st) { - log1.log('[CopilotProvider.transcribeAudio] error=$e\n$st'); + } catch (e) { errorMessage = "Failed to transcribe audio"; return null; } finally { @@ -222,12 +237,15 @@ class CopilotProvider extends ChangeNotifier { if (decoded is String) { return decoded.trim(); } else if (decoded is Map) { - final text = decoded['data'] ?? decoded['text'] ?? decoded['result'] ?? decoded['transcription']; + final map = Map.from(decoded); + final lowerKeyMap = {for (final entry in map.entries) entry.key.toLowerCase(): entry.value}; + final text = lowerKeyMap['transcription'] ?? lowerKeyMap['data'] ?? lowerKeyMap['text'] ?? lowerKeyMap['result']; + if (text is Map) { + return _extractTranscriptionText(jsonEncode(text)); + } if (text != null) return text.toString().trim(); } - } catch (_) { - // Not JSON, fall back to raw body below. - } + } catch (_) {} return trimmedRaw; } @@ -235,8 +253,8 @@ class CopilotProvider extends ChangeNotifier { String _generateSessionId() { final rand = Random(); final bytes = List.generate(16, (_) => rand.nextInt(256)); - bytes[6] = (bytes[6] & 0x0F) | 0x40; // version 4 - bytes[8] = (bytes[8] & 0x3F) | 0x80; // variant 10xx + bytes[6] = (bytes[6] & 0x0F) | 0x40; + bytes[8] = (bytes[8] & 0x3F) | 0x80; String byteHex(int i) => bytes[i].toRadixString(16).padLeft(2, '0'); final hex = List.generate(16, byteHex).join(); return '${hex.substring(0, 8)}-${hex.substring(8, 12)}-${hex.substring(12, 16)}-${hex.substring(16, 20)}-${hex.substring(20, 32)}'; @@ -324,9 +342,339 @@ class CopilotProvider extends ChangeNotifier { } } + Future connectPushToTalkHub() async { + if (_pttHub != null && _pttHub!.state == HubConnectionState.Connected) { + _pttLog('connectPushToTalkHub: already connected, skipping'); + return; + } + await disconnectPushToTalkHub(); + + // final token = URLs.chatApiKey; + final token = ApiManager.instance.user?.token ?? ''; + final userId = ApiManager.instance.user?.id ?? ''; + // final url = "${URLs.copilotHubUrl}?access_token=$token"; + final url = "https://atomsmuat.hmg.com/VoiceWebSocket?access_token=$token"; + // final url = "${URLs.copilotHubUrl}?id=$userId&access_token=$token"; + _pttLog('connectPushToTalkHub: building connection url=$url'); + final httpOp = HttpConnectionOptions(skipNegotiation: false, logMessageContent: true,headers: MessageHeaders()..setHeaderValue('Authorization', 'Bearer $token')); + _pttHub = HubConnectionBuilder().withUrl(url,transportType:HttpTransportType.LongPolling ).withAutomaticReconnect(retryDelays: [2000, 5000, 10000, 20000]).build(); + + _pttHub!.on("RecordingStarted", _onRecordingStarted); + _pttHub!.on("RecordingStopped", _onRecordingStopped); + _pttHub!.on("RecordingCancelled", _onRecordingCancelled); + _pttHub!.on("TranscriptionReceived", _onTranscriptionReceived); + _pttHub!.on("ResponseReceived", _onResponseReceived); + _pttHub!.on("AudioReceived", _onAudioReceived); + _pttHub!.on("RecordingError", _onRecordingError); + + try { + await _pttHub!.start(); + _pttLog('connectPushToTalkHub: connected, state=${_pttHub!.state}'); + } catch (e) { + _pttLog('connectPushToTalkHub: failed error=$e'); + pttErrorMessage = "Failed to connect to push-to-talk service"; + pttStatus = PttStatus.error; + _pttHub = null; + notifyListeners(); + rethrow; + } + } + + Future disconnectPushToTalkHub() async { + _pttLog('disconnectPushToTalkHub: stopping'); + try { + await _abortMicStreaming(); + if (_pttHub != null) { + await _pttHub!.stop(); + } + } catch (e) { + _pttLog('disconnectPushToTalkHub: error=$e'); + } finally { + _pttHub = null; + pttStatus = PttStatus.idle; + } + } + + void startNewPttConversation() { + pttSessionId = null; + pttMessages = []; + pttErrorMessage = null; + pttStatus = PttStatus.idle; + notifyListeners(); + } + + Future beginPushToTalk(String language) async { + if (pttStatus != PttStatus.idle && pttStatus != PttStatus.error) return; + pttSessionId ??= _generateSessionId(); + _pttButtonHeld = true; + pttErrorMessage = null; + pttStatus = PttStatus.connecting; + notifyListeners(); + + try { + if (_pttHub == null || _pttHub!.state != HubConnectionState.Connected) { + await connectPushToTalkHub(); + } + final args = [ + // pttSessionId ?? '', + sessionId ?? '', + ApiManager.instance.user?.id ?? '', + (ApiManager.instance.assetGroup?.id ?? '').toString(), + // language, + "1", + false, + ]; + _pttLog('beginPushToTalk: invoking StartRecording args=$args'); + await _pttHub!.invoke("StartRecording", args: args); + _pttLog('beginPushToTalk: StartRecording invoked, waiting for RecordingStarted event'); + } catch (e) { + _pttLog('beginPushToTalk: failed error=$e'); + pttStatus = PttStatus.error; + pttErrorMessage = "Failed to start recording"; + _pttButtonHeld = false; + notifyListeners(); + } + } + + Future endPushToTalk() async { + _pttButtonHeld = false; + if (pttStatus != PttStatus.recording) { + notifyListeners(); + return; + } + + _pttLog('endPushToTalk: stopping mic and flushing $_pttChunkCount chunks sent so far'); + try { + await _stopMicStreamingAndFlush(); + pttStatus = PttStatus.thinking; + notifyListeners(); + _pttLog('endPushToTalk: invoking StopRecording'); + await _pttHub?.invoke("StopRecording"); + } catch (e) { + _pttLog('endPushToTalk: failed error=$e'); + pttStatus = PttStatus.error; + pttErrorMessage = "Failed to stop recording"; + notifyListeners(); + } + } + + Future cancelPushToTalk() async { + _pttButtonHeld = false; + _pttLog('cancelPushToTalk: invoking CancelRecording'); + try { + await _abortMicStreaming(); + await _pttHub?.invoke("CancelRecording"); + } catch (e) { + _pttLog('cancelPushToTalk: error=$e'); + } + pttStatus = PttStatus.idle; + notifyListeners(); + } + + Future _startMicStreaming() async { + final micStatus = await Permission.microphone.request(); + if (!micStatus.isGranted) { + _pttLog('_startMicStreaming: microphone permission denied'); + pttStatus = PttStatus.error; + pttErrorMessage = "Microphone permission denied"; + notifyListeners(); + await cancelPushToTalk(); + return; + } + + if (!_pttRecorderOpen) { + await _pttRecorder.openRecorder(); + _pttRecorderOpen = true; + } + await _pttRecorder.setSubscriptionDuration(const Duration(milliseconds: 100)); + + _pttMicStreamController = StreamController(); + _pttSendQueue = Future.value(); + _pttChunkCount = 0; + + _pttMicSub = _pttMicStreamController!.stream.listen((chunk) { + _pttChunkCount++; + _pttLog('mic chunk #$_pttChunkCount captured bytes=${chunk.length}'); + _enqueueChunkSend(chunk); + }); + + await _pttRecorder.startRecorder( + toStream: _pttMicStreamController!.sink, + codec: sound.Codec.pcm16, + numChannels: 1, + sampleRate: 48000, + ); + + _pttLog('_startMicStreaming: recorder started'); + pttStatus = PttStatus.recording; + notifyListeners(); + } + + Future _stopMicStreamingAndFlush() async { + try { + if (_pttRecorderOpen) { + await _pttRecorder.stopRecorder(); + } + } catch (e) { + _pttLog('_stopMicStreamingAndFlush: stopRecorder error=$e'); + } + await Future.delayed(const Duration(milliseconds: 50)); + await _pttMicSub?.cancel(); + _pttMicSub = null; + await _pttMicStreamController?.close(); + _pttMicStreamController = null; + await _pttSendQueue; + _pttLog('_stopMicStreamingAndFlush: done, total chunks sent=$_pttChunkCount'); + } + + Future _abortMicStreaming() async { + try { + if (_pttRecorderOpen) { + await _pttRecorder.stopRecorder(); + } + } catch (e) { + _pttLog('_abortMicStreaming: stopRecorder error=$e'); + } + await _pttMicSub?.cancel(); + _pttMicSub = null; + await _pttMicStreamController?.close(); + _pttMicStreamController = null; + _pttSendQueue = Future.value(); + } + + void _enqueueChunkSend(Uint8List chunk) { + _pttSendQueue = _pttSendQueue.then((_) => _sendAudioChunk(chunk)); + } + + Future _sendAudioChunk(Uint8List chunk) async { + try { + await _pttHub?.invoke("SendAudioChunk", args: [base64Encode(chunk)]); + } catch (e) { + _pttLog('_sendAudioChunk: failed error=$e'); + } + } + + void _onRecordingStarted(List? params) async { + _pttLog('event RecordingStarted params=$params'); + try { + if (!_pttButtonHeld) { + _pttLog('RecordingStarted received but button already released, cancelling'); + try { + await _pttHub?.invoke("CancelRecording"); + } catch (_) {} + pttStatus = PttStatus.idle; + notifyListeners(); + return; + } + await _startMicStreaming(); + } catch (e) { + _pttLog('_onRecordingStarted: failed error=$e'); + pttStatus = PttStatus.error; + pttErrorMessage = "Failed to start recording"; + notifyListeners(); + } + } + + void _onRecordingStopped(List? params) { + _pttLog('event RecordingStopped params=$params'); + } + + void _onRecordingCancelled(List? params) { + _pttLog('event RecordingCancelled params=$params'); + pttStatus = PttStatus.idle; + notifyListeners(); + } + + void _onTranscriptionReceived(List? params) { + _pttLog('event TranscriptionReceived params=$params'); + final data = _firstArgAsMap(params); + final text = data?['text']?.toString() ?? ''; + if (text.isEmpty) return; + pttMessages.add(CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.user, content: text, timestamp: DateTime.now())); + notifyListeners(); + } + + void _onResponseReceived(List? params) { + _pttLog('event ResponseReceived params=$params'); + final data = _firstArgAsMap(params); + final text = data?['voiceAnswer']?.toString() ?? ''; + _pttCurrentAssistantMsg = CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.assistant, content: text, timestamp: DateTime.now()); + pttMessages.add(_pttCurrentAssistantMsg!); + notifyListeners(); + } + + void _onAudioReceived(List? params) { + if (params == null || params.isEmpty || params.first == null) return; + try { + final bytes = base64Decode(params.first.toString()); + _pttLog('event AudioReceived bytes=${bytes.length}'); + _pttCurrentAssistantMsg?.ttsAudioClips.add(bytes); + _pttAudioQueue.add(bytes); + notifyListeners(); + _playNextTtsClipIfIdle(); + } catch (e) { + _pttLog('_onAudioReceived: failed to decode error=$e'); + } + } + + void _onRecordingError(List? params) { + final msg = (params != null && params.isNotEmpty) ? params.first?.toString() : null; + _pttLog('event RecordingError message=$msg'); + pttErrorMessage = msg ?? "Recording error occurred"; + pttStatus = PttStatus.error; + _abortMicStreaming(); + notifyListeners(); + } + + Map? _firstArgAsMap(List? params) { + if (params == null || params.isEmpty || params.first == null) return null; + final first = params.first; + if (first is Map) return Map.from(first); + try { + final decoded = jsonDecode(jsonEncode(first)); + if (decoded is Map) return Map.from(decoded); + } catch (_) {} + return null; + } + + Future _playNextTtsClipIfIdle() async { + if (_pttIsPlayingAudio) return; + if (_pttAudioQueue.isEmpty) return; + _pttIsPlayingAudio = true; + pttStatus = PttStatus.playingAudio; + notifyListeners(); + + while (_pttAudioQueue.isNotEmpty) { + final clip = _pttAudioQueue.removeAt(0); + final completer = Completer(); + late StreamSubscription sub; + sub = _pttAudioPlayer.onPlayerComplete.listen((_) { + sub.cancel(); + if (!completer.isCompleted) completer.complete(); + }); + try { + await _pttAudioPlayer.play(audioplayers.BytesSource(clip, mimeType: 'audio/wav')); + await completer.future; + } catch (_) { + sub.cancel(); + } + } + + _pttIsPlayingAudio = false; + pttStatus = PttStatus.idle; + notifyListeners(); + } + + @override void dispose() { _streamSub?.cancel(); + _abortMicStreaming(); + if (_pttRecorderOpen) { + _pttRecorder.closeRecorder(); + } + _pttAudioPlayer.dispose(); + _pttHub?.stop(); super.dispose(); } } diff --git a/lib/modules/copilot/model/copilot_message_model.dart b/lib/modules/copilot/model/copilot_message_model.dart index bfc46dab..8863da5c 100644 --- a/lib/modules/copilot/model/copilot_message_model.dart +++ b/lib/modules/copilot/model/copilot_message_model.dart @@ -50,6 +50,10 @@ class CopilotMessage { // replay their own sent voice message, same as Chat's audio playback. String? localAudioPath; + // Queued TTS audio clips received over the Push-to-Talk SignalR hub + // (one WAV clip per `AudioReceived` event, played back in order). + List ttsAudioClips; + CopilotMessage({ required this.id, required this.role, @@ -61,7 +65,8 @@ class CopilotMessage { this.isStreaming = false, this.isError = false, this.localAudioPath, - }); + List? ttsAudioClips, + }) : ttsAudioClips = ttsAudioClips ?? []; /// Builds a message from GetConversationByThreadId's flat message shape: /// { "role": "human"|"ai", "content": "...", "document_answer": "...", "website_answer": "...", "images": [...] } diff --git a/lib/modules/copilot/screens/copilot_chat_page.dart b/lib/modules/copilot/screens/copilot_chat_page.dart index d599a4b4..3c00448b 100644 --- a/lib/modules/copilot/screens/copilot_chat_page.dart +++ b/lib/modules/copilot/screens/copilot_chat_page.dart @@ -11,6 +11,7 @@ import 'package:test_sa/extensions/string_extensions.dart'; import 'package:test_sa/extensions/text_extensions.dart'; import 'package:test_sa/extensions/widget_extensions.dart'; import 'package:test_sa/modules/copilot/screens/copilot_history_page.dart'; +import 'package:test_sa/modules/copilot/screens/copilot_push_to_talk_page.dart'; import 'package:test_sa/new_views/app_style/app_color.dart'; import 'package:test_sa/new_views/common_widgets/default_app_bar.dart'; import 'package:test_sa/thinking_orbs/src/thinking_orb.dart'; @@ -126,6 +127,13 @@ class _CopilotChatPageState extends State { appBar: DefaultAppBar( title: "ATOMS – AI Assistant", actions: [ + IconButton( + onPressed: () => Navigator.push(context, MaterialPageRoute(builder: (_) => const CopilotPushToTalkPage())), + style: const ButtonStyle(tapTargetSize: MaterialTapTargetSize.shrinkWrap), + icon: Icon(Icons.record_voice_over_rounded, color: AppColor.icon2Color(context)), + constraints: const BoxConstraints(), + tooltip: "Push to Talk", + ), IconButton( onPressed: () => Navigator.push(context, MaterialPageRoute(builder: (_) => const CopilotHistoryPage())), style: const ButtonStyle(tapTargetSize: MaterialTapTargetSize.shrinkWrap), diff --git a/lib/modules/copilot/widgets/copilot_message_bubble.dart b/lib/modules/copilot/widgets/copilot_message_bubble.dart index 5657c89b..0d1d30a8 100644 --- a/lib/modules/copilot/widgets/copilot_message_bubble.dart +++ b/lib/modules/copilot/widgets/copilot_message_bubble.dart @@ -204,6 +204,20 @@ class CopilotMessageBubble extends StatelessWidget { ), ), ], + if (message.ttsAudioClips.isNotEmpty) ...[ + 8.height, + Row( + mainAxisSize: MainAxisSize.min, + children: [ + Icon(Icons.volume_up_rounded, size: 16, color: AppColor.icon2Color(context)), + 4.width, + Text( + "Voice reply", + style: AppTextStyles.bodyText2.copyWith(color: AppColor.textColor(context)), + ), + ], + ), + ], Align( alignment: Alignment.centerRight, widthFactor: 1,