From cc61fa34b871134bf2678110db6d8546ad1e2614 Mon Sep 17 00:00:00 2001 From: WaseemAbbasi22 <50428976+WaseemAbbasi22@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:29:31 +0300 Subject: [PATCH] image upload flow implemented --- lib/controllers/api_routes/api_manager.dart | 82 ++++- .../copilot/controller/copilot_provider.dart | 343 +++++++++--------- lib/modules/copilot/copilot_helper.dart | 51 +++ .../copilot/model/copilot_message_model.dart | 27 +- .../copilot/screens/copilot_chat_page.dart | 53 ++- .../screens/copilot_live_talk_page.dart | 4 - .../widgets/copilot_message_bubble.dart | 23 ++ lib/views/widgets/sound/sound_player.dart | 175 ++++++--- 8 files changed, 483 insertions(+), 275 deletions(-) diff --git a/lib/controllers/api_routes/api_manager.dart b/lib/controllers/api_routes/api_manager.dart index a343c506..a1bb028a 100644 --- a/lib/controllers/api_routes/api_manager.dart +++ b/lib/controllers/api_routes/api_manager.dart @@ -7,6 +7,7 @@ import 'package:flutter/material.dart'; import 'package:fluttertoast/fluttertoast.dart'; import 'package:http/http.dart' as http; import 'package:http/http.dart'; +import 'package:http_parser/http_parser.dart'; import 'package:provider/provider.dart'; import 'package:test_sa/controllers/providers/api/user_provider.dart'; import 'package:test_sa/controllers/providers/settings/setting_provider.dart'; @@ -222,7 +223,10 @@ class ApiManager { request.fields.addAll(body); request.headers.addAll(headers); if (file != null) { - request.files.add(await MultipartFile.fromPath('ImageFile', file.path)); + request.files.add(await MultipartFile.fromPath('imageFile', file.path)); + if (!kReleaseMode) { + print("Attached File: field=imageFile path=${file.path}"); + } } http.StreamedResponse streamedResponse = await _httpClient.send(request); @@ -262,7 +266,24 @@ class ApiManager { var request = http.MultipartRequest('POST', url0); request.fields.addAll(fields); request.headers.addAll(headers); - request.files.add(await http.MultipartFile.fromPath(fileFieldName, file.path, filename: filename)); + final multipartFile = await http.MultipartFile.fromPath( + fileFieldName, + file.path, + filename: filename, + contentType: _contentTypeForPath(file.path), + ); + request.files.add(multipartFile); + + if (!kReleaseMode) { + print("===== MULTIPART REQUEST ====="); + print("URL: $url0"); + print("Method: ${request.method}"); + print("Request Headers: ${request.headers}"); + print("Fields: ${request.fields}"); + print("File -> field: '$fileFieldName', filename: '${multipartFile.filename}', " + "length: ${multipartFile.length} bytes, contentType: ${multipartFile.contentType}, path: ${file.path}"); + print("============================="); + } http.StreamedResponse streamedResponse = await _httpClient.send(request); @@ -271,9 +292,66 @@ class ApiManager { showLoginDialog(); } + // On any error status, read + log the response body so the exact server + // message (e.g. a 400 validation error) is visible for backend debugging. + if (!kReleaseMode && (streamedResponse.statusCode < 200 || streamedResponse.statusCode >= 300)) { + final errorBody = await streamedResponse.stream.bytesToString(); + print("===== MULTIPART RESPONSE ERROR ====="); + print("Status: ${streamedResponse.statusCode} ${streamedResponse.reasonPhrase}"); + print("Response Headers: ${streamedResponse.headers}"); + print("Response Body: $errorBody"); + print("===================================="); + // Return a fresh StreamedResponse carrying the already-read body so the + // caller can still inspect it (the original stream is now drained). + return http.StreamedResponse( + Stream.value(utf8.encode(errorBody)), + streamedResponse.statusCode, + reasonPhrase: streamedResponse.reasonPhrase, + headers: streamedResponse.headers, + ); + } + return streamedResponse; } + /// Infers the multipart `Content-Type` from a file's extension so the server + /// receives e.g. `image/png` instead of the generic `application/octet-stream` + /// (which backends reject with "must be an image"). Returns null for unknown + /// extensions so the http package falls back to its default. + MediaType? _contentTypeForPath(String path) { + final ext = path.split('.').last.toLowerCase(); + switch (ext) { + case 'png': + return MediaType('image', 'png'); + case 'jpg': + case 'jpeg': + return MediaType('image', 'jpeg'); + case 'gif': + return MediaType('image', 'gif'); + case 'webp': + return MediaType('image', 'webp'); + case 'bmp': + return MediaType('image', 'bmp'); + case 'heic': + return MediaType('image', 'heic'); + case 'heif': + return MediaType('image', 'heif'); + // Audio types (used by the voice/transcription flows). + case 'wav': + return MediaType('audio', 'wav'); + case 'mp3': + return MediaType('audio', 'mpeg'); + case 'm4a': + return MediaType('audio', 'mp4'); + case 'aac': + return MediaType('audio', 'aac'); + case 'ogg': + return MediaType('audio', 'ogg'); + default: + return null; + } + } + /// POST FormData request with a file attachment that waits for and returns the /// full [http.Response] (use this when the endpoint returns a plain/JSON body /// instead of a stream, e.g. audio transcription). diff --git a/lib/modules/copilot/controller/copilot_provider.dart b/lib/modules/copilot/controller/copilot_provider.dart index 5d82fafa..23b9f14e 100644 --- a/lib/modules/copilot/controller/copilot_provider.dart +++ b/lib/modules/copilot/controller/copilot_provider.dart @@ -13,13 +13,11 @@ import 'package:http/http.dart'; import 'package:permission_handler/permission_handler.dart'; import 'package:test_sa/controllers/api_routes/api_manager.dart'; import 'package:test_sa/controllers/api_routes/urls.dart'; - import '../model/copilot_message_model.dart'; import '../utils/ptt_signalr_http_client.dart'; enum PttStatus { idle, connecting, recording, thinking, playingAudio, error } -/// Connection state of the push-to-talk SignalR hub (observable by the UI). enum PttHubState { disconnected, connecting, connected } class CopilotProvider extends ChangeNotifier { @@ -35,9 +33,6 @@ class CopilotProvider extends ChangeNotifier { HubConnection? _pttHub; PttSignalRHttpClient? _pttHttpClient; - - // Serializes connect/disconnect so they never overlap when the user navigates - // in and out of the Push-to-Talk screen quickly. Future _pttOpQueue = Future.value(); PttStatus pttStatus = PttStatus.idle; PttHubState pttHubState = PttHubState.disconnected; @@ -55,28 +50,23 @@ class CopilotProvider extends ChangeNotifier { StreamSubscription? _pttMicSub; Future _pttSendQueue = Future.value(); int _pttChunkCount = 0; - - // Live Talk voice-activity (silence) detection, computed from the PCM mic - // chunks (onProgress metering is not emitted when recording to a stream). - DateTime? _pttLastLoudAt; + DateTime? _pttLastSpeechAt; + DateTime? _pttSpeechCandidateAt; + DateTime? _pttLastAudioChunkAt; + DateTime? _pttCalibrationStartedAt; + Timer? _pttSilenceTimer; bool _pttAutoStopTriggered = false; bool _pttHasDetectedSpeech = false; - // Self-calibrating VAD: we continuously track the quiet "floor" AND the loud - // speech "peak", then put the speaking threshold in the gap between them, so - // detection adapts to any device mic gain / ambient noise level. - double _pttNoiseFloor = 0.01; - double _pttSpeechPeak = 0.05; - // Smoothed voice-activity (0..1 moving average of "is this chunk speech?"). - // Brief room-noise/echo spikes can't reset the silence timer — only sustained - // speech keeps this high, which is what we key the auto-stop off of. - double _pttVoiceActivity = 0.0; - static const Duration _silenceHoldDuration = Duration(milliseconds: 3000); - + bool _pttDetectorActive = false; + bool _pttCalibrated = false; + double _pttNoiseFloor = 0.0; + double _pttCalibrationSum = 0.0; + int _pttCalibrationSamples = 0; + static const Duration _silenceHoldDuration = Duration(seconds: 3); + static const Duration _minimumSpeechDuration = Duration(milliseconds: 300); + static const Duration _calibrationDuration = Duration(milliseconds: 700); + static const Duration _audioChunkTimeout = Duration(seconds: 2); CopilotMessage? _pttCurrentAssistantMsg; - - // Live Talk: the provider plays the bot's TTS audio itself so it can resume - // listening only AFTER the bot has finished speaking (avoids recording the - // bot's own voice). Push-to-Talk does not use any of this. final audioplayers.AudioPlayer _livePlayer = audioplayers.AudioPlayer(); final List _liveAudioQueue = []; bool _liveIsPlaying = false; @@ -229,6 +219,38 @@ class CopilotProvider extends ChangeNotifier { } } + Future sendImageFile(File imageFile, String query, String language) async { + if (isStreaming) return; + sessionId ??= _generateSessionId(); + final humanImageMsg = CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.user, content: query, localImagePath: imageFile.path, timestamp: DateTime.now()); + messages.add(humanImageMsg); + final assistantMsg = CopilotMessage(id: UniqueKey().toString(), role: CopilotRole.assistant, isStreaming: true, timestamp: DateTime.now()); + messages.add(assistantMsg); + isStreaming = true; + errorMessage = null; + notifyListeners(); + + try { + final streamedResponse = await ApiManager.instance.postFormDataStreamingWithFile( + URLs.copilotQueryStream, + fields: { + "sessionId": sessionId!, + "isImage": "true", + "isAudio": "false", + "langId": "$language", + 'isFromMobile': 'true', + "query": query, + }, + fileFieldName: "ImageFile", + file: imageFile, + filename: imageFile.path.split('/').last, + ); + await _consumeStream(streamedResponse, assistantMsg, null); + } catch (e) { + _onStreamError(assistantMsg, e); + } + } + Future transcribeAudio(File audioFile, String language) async { isTranscribing = true; errorMessage = null; @@ -383,9 +405,6 @@ class CopilotProvider extends ChangeNotifier { } } - /// Builds a brand-new push-to-talk connection. Any previous connection is - /// torn down first, so entering the screen always gives a fresh connection. - /// Serialized via [_pttOpQueue] so it never races with a disconnect. Future connectPushToTalkHub() { _pttOpQueue = _pttOpQueue.then((_) => _doConnectPushToTalkHub()).catchError((e) { _pttLog('connectPushToTalkHub: queued error=$e'); @@ -394,63 +413,23 @@ class CopilotProvider extends ChangeNotifier { } Future _doConnectPushToTalkHub() async { - // Always start from a clean slate so every entry is a fresh connection. await _stopHubAndCleanup(); _setHubState(PttHubState.connecting); - // signalr_netcore can throw a "Null check operator used on a null value" - // from HubConnection._startInternal when the server closes the long-poll - // connection mid-handshake (a package race on _handshakeCompleter). It is - // transient, so retry a few times before surfacing an error. - const maxAttempts = 3; - Object? lastError; - for (var attempt = 1; attempt <= maxAttempts; attempt++) { - try { - await _buildAndStartHub(); - return; // connected - } catch (e, st) { - lastError = e; - _pttLog('connectPushToTalkHub: attempt $attempt/$maxAttempts failed error=$e'); - await _stopHubAndCleanup(); - if (attempt < maxAttempts) { - await Future.delayed(Duration(milliseconds: 400 * attempt)); - } else { - _pttLog('connectPushToTalkHub: giving up\n$st'); - } - } - } - - pttErrorMessage = "Failed to connect to push-to-talk service"; - pttStatus = PttStatus.error; - pttHubState = PttHubState.disconnected; - notifyListeners(); - throw lastError ?? StateError('Failed to connect'); - } - - /// Builds a single fresh hub and starts it. Throws on failure so the caller - /// ([_doConnectPushToTalkHub]) can retry. - Future _buildAndStartHub() async { final token = ApiManager.instance.user?.token ?? ''; final url = "${URLs.copilotHubUrl}?access_token=$token"; _pttLog('connectPushToTalkHub: building connection url=${URLs.copilotHubUrl}'); final httpClient = PttSignalRHttpClient(); _pttHttpClient = httpClient; final httpOp = HttpConnectionOptions( - // Server only accepts LongPolling. transport: HttpTransportType.LongPolling, skipNegotiation: false, - // Never log message content: it contains base64 audio and is very heavy. logMessageContent: false, headers: MessageHeaders()..setHeaderValue('Authorization', 'Bearer $token'), - // Ensures the Authorization header is also sent on poll + the DELETE - // that tells the server to drop the connection when we stop. accessTokenFactory: () async => token, - // Closes sockets properly and truly aborts the pending long-poll on stop - // (signalr_netcore's default client leaks them). httpClient: httpClient, ); final hub = HubConnectionBuilder().withUrl(url, options: httpOp).withAutomaticReconnect(retryDelays: [0, 2000, 5000, 10000, 20000]).build(); - // More tolerant timeouts for mobile networks (defaults are 30s / 15s). hub.serverTimeoutInMilliseconds = 60000; hub.keepAliveIntervalInMilliseconds = 15000; _pttHub = hub; @@ -465,8 +444,6 @@ class CopilotProvider extends ChangeNotifier { if (!identical(hub, _pttHub)) return; _pttLog('hub reconnected connectionId=$connectionId'); pttHubState = PttHubState.connected; - // Server-side recording state is bound to the old connection id, so the - // user must start a new turn. if (pttStatus == PttStatus.error || pttStatus == PttStatus.connecting) { pttStatus = PttStatus.idle; } @@ -480,23 +457,33 @@ class CopilotProvider extends ChangeNotifier { _handleConnectionInterrupted("Connection lost. Tap retry to reconnect."); }); - hub.on("RecordingStarted", _onRecordingStarted); - hub.on("RecordingStopped", _onRecordingStopped); - hub.on("RecordingCancelled", _onRecordingCancelled); - hub.on("TranscriptionReceived", _onTranscriptionReceived); - hub.on("ResponseReceived", _onResponseReceived); - hub.on("AudioReceived", _onAudioReceived); - hub.on("RecordingError", _onRecordingError); - - _pttLog('connectPushToTalkHub: calling start()...'); - await hub.start()?.timeout(const Duration(seconds: 20), onTimeout: () { - throw TimeoutException('Connection to push-to-talk service timed out'); - }); - _pttLog('connectPushToTalkHub: connected, state=${hub.state}'); - pttHubState = PttHubState.connected; - if (pttStatus == PttStatus.error) pttStatus = PttStatus.idle; - pttErrorMessage = null; - notifyListeners(); + _pttHub!.on("RecordingStarted", _onRecordingStarted); + _pttHub!.on("RecordingStopped", _onRecordingStopped); + _pttHub!.on("RecordingCancelled", _onRecordingCancelled); + _pttHub!.on("TranscriptionReceived", _onTranscriptionReceived); + _pttHub!.on("ResponseReceived", _onResponseReceived); + _pttHub!.on("AudioReceived", _onAudioReceived); + _pttHub!.on("RecordingError", _onRecordingError); + + try { + _pttLog('connectPushToTalkHub: calling start()...'); + await _pttHub!.start()?.timeout(const Duration(seconds: 20), onTimeout: () { + throw TimeoutException('Connection to push-to-talk service timed out'); + }); + _pttLog('connectPushToTalkHub: connected, state=${_pttHub!.state}'); + pttHubState = PttHubState.connected; + if (pttStatus == PttStatus.error) pttStatus = PttStatus.idle; + pttErrorMessage = null; + notifyListeners(); + } catch (e, st) { + _pttLog('connectPushToTalkHub: failed error=$e\n$st'); + pttErrorMessage = "Failed to connect to push-to-talk service"; + pttStatus = PttStatus.error; + await _stopHubAndCleanup(); + pttHubState = PttHubState.disconnected; + notifyListeners(); + rethrow; + } } void _setHubState(PttHubState state) { @@ -505,8 +492,6 @@ class CopilotProvider extends ChangeNotifier { notifyListeners(); } - /// Disconnects the push-to-talk connection cleanly. Called when leaving the - /// screen. Serialized via [_pttOpQueue] so it never races with a connect. Future disconnectPushToTalkHub() { _pttOpQueue = _pttOpQueue.then((_) => _doDisconnectPushToTalkHub()).catchError((e) { _pttLog('disconnectPushToTalkHub: queued error=$e'); @@ -520,11 +505,8 @@ class CopilotProvider extends ChangeNotifier { pttStatus = PttStatus.idle; } - /// Stops mic streaming and tears down the hub. Does NOT touch [_pttOpQueue] - /// so it is safe to call from inside queued operations. Future _stopHubAndCleanup() async { _cancelResponseWatchdog(); - // Detach first so callbacks (onclose) from this hub are ignored. final hub = _pttHub; _pttHub = null; final httpClient = _pttHttpClient; @@ -540,14 +522,10 @@ class CopilotProvider extends ChangeNotifier { } catch (e) { _pttLog('_stopHubAndCleanup: error=$e'); } finally { - // Kill any request still open (e.g. a pending long-poll) so no orphan - // connections pile up between visits to the screen. httpClient?.closeAll(); } } - /// Called when the hub drops (reconnecting/closed). Stops the mic and frees - /// the UI so it never stays stuck in recording/thinking forever. void _handleConnectionInterrupted(String message) { _cancelResponseWatchdog(); _pttButtonHeld = false; @@ -560,8 +538,6 @@ class CopilotProvider extends ChangeNotifier { notifyListeners(); } - /// Re-establishes the connection if it was lost (e.g. after the app was - /// in background). Safe to call any time. Future ensurePushToTalkConnected() async { final state = _pttHub?.state; if (state == HubConnectionState.Connected || state == HubConnectionState.Connecting || state == HubConnectionState.Reconnecting) return; @@ -643,25 +619,20 @@ class CopilotProvider extends ChangeNotifier { } } - /// Starts a continuous "Live Talk" session: records, waits for a response, - /// then automatically starts listening again until [stopLiveTalk] is called. Future startLiveTalk(String language) async { if (pttStatus != PttStatus.idle && pttStatus != PttStatus.error) return; liveSessionActive = true; await beginPushToTalk(language, isLive: true); } - /// Stops the current Live Talk session completely. The button is only used to - /// start/stop the whole session, so ending it must hard-stop everything: - /// no more listening, thinking, or speaking until the user taps to start again. Future stopLiveTalk() async { - _pttLog('stopLiveTalk: ending session, stopping everything'); liveSessionActive = false; - _pttIsLive = false; - _pttAutoStopTriggered = true; // block any in-flight auto-stop - _cancelResponseWatchdog(); _stopLivePlayback(); - await cancelPushToTalk(); + if (pttStatus == PttStatus.recording) { + await endPushToTalk(); + } else { + await cancelPushToTalk(); + } } Future endPushToTalk() async { @@ -738,93 +709,119 @@ class CopilotProvider extends ChangeNotifier { if (_pttIsLive) _processSilenceDetection(chunk); }); - await _pttRecorder.startRecorder( - toStream: _pttMicStreamController!.sink, - codec: sound.Codec.pcm16, - numChannels: 1, - sampleRate: 48000, - ); + if (_pttIsLive) { + _startSilenceDetection(); + } + + try { + await _pttRecorder.startRecorder( + toStream: _pttMicStreamController!.sink, + codec: sound.Codec.pcm16, + numChannels: 1, + sampleRate: 48000, + ); + } catch (_) { + _stopSilenceDetection(); + rethrow; + } _pttLog('_startMicStreaming: recorder started'); pttStatus = PttStatus.recording; notifyListeners(); - - if (_pttIsLive) { - _startSilenceDetection(); - } } - /// Resets voice-activity state for a new Live Talk recording. Actual silence - /// detection happens per mic chunk in [_processSilenceDetection]. void _startSilenceDetection() { + _pttSilenceTimer?.cancel(); _pttAutoStopTriggered = false; _pttHasDetectedSpeech = false; - _pttLastLoudAt = null; - _pttNoiseFloor = 0.01; - _pttSpeechPeak = 0.05; - _pttVoiceActivity = 0.0; + _pttDetectorActive = true; + _pttCalibrated = false; + _pttLastSpeechAt = null; + _pttSpeechCandidateAt = null; + _pttLastAudioChunkAt = null; + _pttNoiseFloor = 0; + _pttCalibrationSum = 0; + _pttCalibrationSamples = 0; + _pttCalibrationStartedAt = null; + + _pttSilenceTimer = Timer.periodic( + const Duration(milliseconds: 100), + (_) => _checkSpeechSilence(), + ); } void _stopSilenceDetection() { - _pttLastLoudAt = null; - _pttHasDetectedSpeech = false; + _pttSilenceTimer?.cancel(); + _pttSilenceTimer = null; + _pttDetectorActive = false; _pttAutoStopTriggered = false; - _pttVoiceActivity = 0.0; + _pttHasDetectedSpeech = false; + _pttLastSpeechAt = null; + _pttSpeechCandidateAt = null; + _pttLastAudioChunkAt = null; + _pttCalibrationStartedAt = null; } - /// Live Talk auto-stop: once the user has spoken and then stayed silent for - /// [_silenceHoldDuration] (3s), automatically send StopRecording — the button - /// is only used to start/stop the whole session, never per-utterance. - /// - /// Self-calibrating + noise/echo tolerant: - /// - Tracks the quiet floor (fast down / slow up) and the speech peak - /// (fast up / slow down), and sets a per-chunk threshold 45% of the way - /// from floor toward peak. - /// - Smooths that per-chunk decision into [_pttVoiceActivity] so a few - /// stray room-noise / echo chunks can't masquerade as speech and keep - /// resetting the silence countdown. Only *sustained* speech counts. void _processSilenceDetection(Uint8List chunk) { - if (_pttAutoStopTriggered) return; - final rms = _chunkRms(chunk); - final now = DateTime.now(); - - if (rms < _pttNoiseFloor) { - _pttNoiseFloor = rms; - } else { - _pttNoiseFloor = _pttNoiseFloor * 0.995 + rms * 0.005; + if (!_pttDetectorActive || _pttAutoStopTriggered || pttStatus != PttStatus.recording || !liveSessionActive) { + return; } - if (rms > _pttSpeechPeak) { - _pttSpeechPeak = rms; - } else { - _pttSpeechPeak = _pttSpeechPeak * 0.995 + rms * 0.005; + + final now = DateTime.now(); + final rms = _chunkRms(chunk); + if (!rms.isFinite) return; + _pttLastAudioChunkAt = now; + + if (!_pttCalibrated) { + _pttCalibrationStartedAt ??= now; + _pttCalibrationSum += rms; + _pttCalibrationSamples++; + if (now.difference(_pttCalibrationStartedAt!) < _calibrationDuration) { + return; + } + _pttNoiseFloor = (_pttCalibrationSum / _pttCalibrationSamples).clamp(0.001, 0.15).toDouble(); + _pttCalibrated = true; + return; } - final span = _pttSpeechPeak - _pttNoiseFloor; - final threshold = max(_pttNoiseFloor + span * 0.45, 0.012); - final chunkIsSpeech = rms > threshold && span > 0.01; + final speechThreshold = max(0.008, _pttNoiseFloor * 2.5); + final isSpeechCandidate = rms > speechThreshold; - // Smooth: a single loud chunk only nudges activity, sustained speech climbs. - _pttVoiceActivity = _pttVoiceActivity * 0.8 + (chunkIsSpeech ? 1.0 : 0.0) * 0.2; - // Hysteresis: need clear sustained energy to count as actively speaking. - final activelySpeaking = _pttVoiceActivity > 0.4; + if (isSpeechCandidate) { + _pttSpeechCandidateAt ??= now; + if (now.difference(_pttSpeechCandidateAt!) >= _minimumSpeechDuration) { + _pttHasDetectedSpeech = true; + _pttLastSpeechAt = now; + } + return; + } - _pttLog('silence: rms=${rms.toStringAsFixed(4)} floor=${_pttNoiseFloor.toStringAsFixed(4)} peak=${_pttSpeechPeak.toStringAsFixed(4)} thr=${threshold.toStringAsFixed(4)} act=${_pttVoiceActivity.toStringAsFixed(2)} speaking=$activelySpeaking spoke=$_pttHasDetectedSpeech'); + _pttSpeechCandidateAt = null; + if (!_pttHasDetectedSpeech) { + _pttNoiseFloor = (_pttNoiseFloor * 0.95 + rms * 0.05).clamp(0.001, 0.15).toDouble(); + } + } - if (activelySpeaking) { - _pttLastLoudAt = now; - _pttHasDetectedSpeech = true; + void _checkSpeechSilence() { + if (!_pttDetectorActive || _pttAutoStopTriggered || !_pttHasDetectedSpeech || pttStatus != PttStatus.recording || !liveSessionActive) { return; } - if (!_pttHasDetectedSpeech) return; - final lastLoud = _pttLastLoudAt; - if (lastLoud != null && now.difference(lastLoud) >= _silenceHoldDuration) { - _pttAutoStopTriggered = true; - _pttLog('live: silence after speech, auto-stopping recording'); - endPushToTalk().catchError((_) {}); - } + + final now = DateTime.now(); + final lastChunk = _pttLastAudioChunkAt; + if (lastChunk == null || now.difference(lastChunk) > _audioChunkTimeout) return; + + final lastSpeech = _pttLastSpeechAt; + if (lastSpeech == null || now.difference(lastSpeech) < _silenceHoldDuration) return; + + _pttAutoStopTriggered = true; + _pttDetectorActive = false; + _pttSilenceTimer?.cancel(); + _pttSilenceTimer = null; + _pttLog('live: silence after speech, auto-stopping recording'); + endPushToTalk().catchError((_) {}); } - /// Root-mean-square amplitude (0..1) of a PCM16 mono little-endian chunk. double _chunkRms(Uint8List chunk) { final sampleCount = chunk.length ~/ 2; if (sampleCount == 0) return 0; @@ -934,9 +931,6 @@ class CopilotProvider extends ChangeNotifier { pttMessages.add(_pttCurrentAssistantMsg!); if (liveSessionActive && _pttIsLive) { - // Live Talk: let the bot finish speaking its TTS audio, then resume - // listening. _maybeRestartLiveListening is debounced so it waits for any - // audio clips that arrive right after this text response. pttStatus = PttStatus.playingAudio; notifyListeners(); _maybeRestartLiveListening(); @@ -952,8 +946,6 @@ class CopilotProvider extends ChangeNotifier { final bytes = base64Decode(params.first.toString()); _pttLog('event AudioReceived bytes=${bytes.length}'); if (liveSessionActive && _pttIsLive) { - // Live Talk: queue and play through the provider so we know when the - // bot finished speaking (to resume listening). No widget player. _liveAudioQueue.add(bytes); _playNextLiveClip(); } else { @@ -965,7 +957,7 @@ class CopilotProvider extends ChangeNotifier { } } - // ----- Live Talk audio playback + auto-resume listening ----- +// ----- Live Talk audio playback + auto-resume listening ----- void _playNextLiveClip() { if (_liveIsPlaying || _liveAudioQueue.isEmpty) return; @@ -991,8 +983,6 @@ class CopilotProvider extends ChangeNotifier { } } - /// After the bot stops speaking, resume listening. Debounced so trailing - /// audio clips (arriving just after) don't cause a premature restart. void _maybeRestartLiveListening() { if (!liveSessionActive || !_pttIsLive) return; _liveRestartTimer?.cancel(); @@ -1041,6 +1031,7 @@ class CopilotProvider extends ChangeNotifier { void dispose() { _streamSub?.cancel(); _cancelResponseWatchdog(); + _stopSilenceDetection(); _liveRestartTimer?.cancel(); _livePlayer.dispose(); _abortMicStreaming(); diff --git a/lib/modules/copilot/copilot_helper.dart b/lib/modules/copilot/copilot_helper.dart index a510e2b7..543fa1da 100644 --- a/lib/modules/copilot/copilot_helper.dart +++ b/lib/modules/copilot/copilot_helper.dart @@ -25,6 +25,57 @@ class CopilotHelper { return _pickImage(context, pickMode: pickMode, allowGallery: true); } + static Future pickImageFile(BuildContext context) async { + final _ImagePickSource? source = await _showImageSourceSheet(context); + if (source == null) return null; + if (!context.mounted) return null; + + if (source == _ImagePickSource.files) { + final FilePickerResult? result = await FilePicker.platform.pickFiles( + type: FileType.image, + allowMultiple: false, + ); + final String? path = result?.files.single.path; + if (path == null) return null; + return File(path); + } + + if (source == _ImagePickSource.gallery) { + return _pickXFile(ImageSource.gallery); + } + + // Camera: ensure permission first. + final status = await Permission.camera.status; + if (status.isGranted) { + return _pickXFile(ImageSource.camera); + } + + final permission = await Permission.camera.request(); + if (permission.isGranted) { + return _pickXFile(ImageSource.camera); + } + + if (permission.isPermanentlyDenied) { + await openAppSettings(); + return null; + } + + debugPrint('Camera permission denied'); + return null; + } + + /// Picks an [XFile] from [source] and returns it as a [File] (no upload). + static Future _pickXFile(ImageSource source) async { + final XFile? pickedFile = await ImagePicker().pickImage( + source: source, + imageQuality: 70, + ); + if (pickedFile == null) return null; + final File imageFile = File(pickedFile.path); + debugPrint('CopilotHelper: picked image extension = ${imageFile.path.split('.').last}'); + return imageFile; + } + static Future _pickImage(BuildContext context, {bool pickMode = false, bool allowGallery = false}) async { _ImagePickSource source = _ImagePickSource.camera; diff --git a/lib/modules/copilot/model/copilot_message_model.dart b/lib/modules/copilot/model/copilot_message_model.dart index 8863da5c..ad635ad8 100644 --- a/lib/modules/copilot/model/copilot_message_model.dart +++ b/lib/modules/copilot/model/copilot_message_model.dart @@ -1,10 +1,7 @@ import 'package:flutter/foundation.dart'; -/// Role of a message inside a Copilot conversation. enum CopilotRole { user, assistant } -/// A single reference image returned alongside an assistant answer -/// (e.g. a scanned page from a referenced document). class CopilotImage { final String base64; final String source; @@ -29,29 +26,18 @@ class CopilotImage { ); } -/// A single message (human query or assistant answer) inside a Copilot conversation. -/// -/// This is intentionally a NEW, small model (not a reuse of chat's SingleUserChatModel), -/// since the Copilot Automation API response shape (answer/document_answer/website_answer/images) -/// is completely different from the multi-user chat history model. class CopilotMessage { final String id; final CopilotRole role; - String content; // assistant answer text OR the human query text + String content; String? documentAnswer; String? websiteAnswer; List images; DateTime? timestamp; bool isStreaming; bool isError; - - // Local path of a recorded audio file for a user's voice query (never a remote - // download URL - the backend never sends audio back). Used to let the user - // replay their own sent voice message, same as Chat's audio playback. String? localAudioPath; - - // Queued TTS audio clips received over the Push-to-Talk SignalR hub - // (one WAV clip per `AudioReceived` event, played back in order). + String? localImagePath; List ttsAudioClips; CopilotMessage({ @@ -65,14 +51,10 @@ class CopilotMessage { this.isStreaming = false, this.isError = false, this.localAudioPath, + this.localImagePath, List? ttsAudioClips, }) : ttsAudioClips = ttsAudioClips ?? []; - /// Builds a message from GetConversationByThreadId's flat message shape: - /// { "role": "human"|"ai", "content": "...", "document_answer": "...", "website_answer": "...", "images": [...] } - /// NOTE: the API currently always returns `timestamp: null`, so we fall back to - /// "now" purely so the UI has a value to render (matches Chat's bubble layout, - /// which always shows a time under each message). factory CopilotMessage.fromHistoryJson(Map json) => CopilotMessage( id: UniqueKey().toString(), role: json['role'] == 'ai' ? CopilotRole.assistant : CopilotRole.user, @@ -84,8 +66,6 @@ class CopilotMessage { ); } -/// A single event decoded from a QueryStream SSE `data:` line -/// (payload shape: {"type":"response","content":{...}}, or the literal string "[DONE]"). class CopilotStreamEvent { final String type; final CopilotStreamContent? content; @@ -124,7 +104,6 @@ class CopilotStreamContent { ); } -/// A single row in the Copilot query history list (GetQueryHistory). class CopilotHistoryItem { final String threadId; final String firstHumanMessage; diff --git a/lib/modules/copilot/screens/copilot_chat_page.dart b/lib/modules/copilot/screens/copilot_chat_page.dart index 80e5a312..524652d7 100644 --- a/lib/modules/copilot/screens/copilot_chat_page.dart +++ b/lib/modules/copilot/screens/copilot_chat_page.dart @@ -10,10 +10,10 @@ import 'package:test_sa/extensions/int_extensions.dart'; import 'package:test_sa/extensions/string_extensions.dart'; import 'package:test_sa/extensions/text_extensions.dart'; import 'package:test_sa/extensions/widget_extensions.dart'; +import 'package:test_sa/modules/copilot/copilot_helper.dart'; import 'package:test_sa/modules/copilot/screens/copilot_history_page.dart'; import 'package:test_sa/modules/copilot/screens/copilot_live_talk_page.dart'; import 'package:test_sa/modules/copilot/screens/copilot_push_to_talk_page.dart'; -import 'package:test_sa/modules/cx_module/chat/helper/chat_file_picker.dart'; import 'package:test_sa/new_views/app_style/app_color.dart'; import 'package:test_sa/new_views/common_widgets/default_app_bar.dart'; import 'package:test_sa/thinking_orbs/src/thinking_orb.dart'; @@ -76,7 +76,13 @@ class _CopilotChatPageState extends State { final text = _textController.text.trim(); if (text.isEmpty) return; _textController.clear(); - context.read().sendText(text, context.settingProvider.language!, image); + final provider = context.read(); + final language = context.settingProvider.language!; + if (image != null) { + provider.sendImageFile(image!, text, language); + } else { + provider.sendText(text, language, null); + } image = null; setState(() {}); _scrollToBottom(); @@ -319,31 +325,22 @@ class _CopilotChatPageState extends State { hintStyle: TextStyle(fontWeight: FontWeight.w500, fontSize: context.isTablet() ? 16 : 12, color: AppColor.lightTextColor(context)), ), ).expanded, - // if (image == null) - // IconButton( - // onPressed: () async { - // FocusScope.of(context).unfocus(); - // File? file = (await showModalBottomSheet( - // context: context, - // backgroundColor: AppColor.bottomSheetColor(context), - // shape: const RoundedRectangleBorder( - // borderRadius: BorderRadius.vertical( - // top: Radius.circular(20), - // ), - // ), - // clipBehavior: Clip.antiAliasWithSaveLayer, - // builder: (BuildContext context) => ChatFilePicker())) as File?; - // if (file != null) { - // image = file; - // setState(() {}); - // } - // }, - // style: const ButtonStyle( - // tapTargetSize: MaterialTapTargetSize.shrinkWrap, // or .padded - // ), - // icon: "chat_attachment".toSvgAsset(width: 24, height: 24, color: AppColor.icon2Color(context)), - // constraints: const BoxConstraints(), - // ), + if (image == null) + IconButton( + onPressed: () async { + FocusScope.of(context).unfocus(); + final File? file = await CopilotHelper.pickImageFile(context); + if (file != null) { + image = file; + setState(() {}); + } + }, + style: const ButtonStyle( + tapTargetSize: MaterialTapTargetSize.shrinkWrap, // or .padded + ), + icon: "chat_attachment".toSvgAsset(width: 24, height: 24, color: AppColor.icon2Color(context)), + constraints: const BoxConstraints(), + ), Consumer( builder: (_, provider, __) => IconButton( onPressed: provider.isTranscribing ? null : _startRecording, @@ -517,4 +514,4 @@ class _ShimmerBubble extends StatelessWidget { ), ); } -} +} \ No newline at end of file diff --git a/lib/modules/copilot/screens/copilot_live_talk_page.dart b/lib/modules/copilot/screens/copilot_live_talk_page.dart index 8d499d39..9b9eece3 100644 --- a/lib/modules/copilot/screens/copilot_live_talk_page.dart +++ b/lib/modules/copilot/screens/copilot_live_talk_page.dart @@ -99,10 +99,7 @@ class _CopilotLiveTalkPageState extends State with WidgetsB } } - /// Shown in place of the mic while the socket is connecting / disconnected. Widget _buildConnectionPlaceholder(BuildContext context, CopilotProvider provider) { - // Treat "disconnected without an error" as connecting (e.g. the first frame - // before the connection attempt kicks off). final isConnecting = provider.pttHubState == PttHubState.connecting || provider.pttErrorMessage == null; return SizedBox( height: 148, @@ -187,7 +184,6 @@ class _CopilotLiveTalkPageState extends State with WidgetsB SafeArea( child: Consumer( builder: (context, provider, _) { - // Hide the mic until the socket connection is established. if (provider.pttHubState != PttHubState.connected) { return _buildConnectionPlaceholder(context, provider); } diff --git a/lib/modules/copilot/widgets/copilot_message_bubble.dart b/lib/modules/copilot/widgets/copilot_message_bubble.dart index 2e6f3f2c..f05ab0b4 100644 --- a/lib/modules/copilot/widgets/copilot_message_bubble.dart +++ b/lib/modules/copilot/widgets/copilot_message_bubble.dart @@ -1,4 +1,5 @@ import 'dart:convert'; +import 'dart:io'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter_markdown_plus/flutter_markdown_plus.dart'; @@ -76,6 +77,28 @@ class CopilotMessageBubble extends StatelessWidget { mainAxisSize: MainAxisSize.min, crossAxisAlignment: CrossAxisAlignment.end, children: [ + if (message.localImagePath != null) ...[ + ClipRRect( + borderRadius: BorderRadius.circular(8), + child: GestureDetector( + onTap: () => context.fullScreenImageViewer(Image.file(File(message.localImagePath!))), + child: Image.file(File(message.localImagePath!), width: 120, height: 120, fit: BoxFit.cover), + ), + ), + 6.height, + ], + if(message.images.isNotEmpty) ...[ + for (var imagePath in message.images) ...[ + ClipRRect( + borderRadius: BorderRadius.circular(8), + child: GestureDetector( + onTap: () => context.fullScreenImageViewer(Image.memory(base64Decode(imagePath.base64))), + child: Image.memory(base64Decode(imagePath.base64), width: 120, height: 120, fit: BoxFit.cover), + ), + ), + 6.height, + ], + ], if (message.localAudioPath != null) ASoundPlayer(audio: message.localAudioPath!) else diff --git a/lib/views/widgets/sound/sound_player.dart b/lib/views/widgets/sound/sound_player.dart index 93a25a01..e21e2ac2 100644 --- a/lib/views/widgets/sound/sound_player.dart +++ b/lib/views/widgets/sound/sound_player.dart @@ -1,10 +1,8 @@ import 'package:audioplayers/audioplayers.dart'; import 'package:flutter/material.dart'; -import 'package:test_sa/extensions/context_extension.dart'; import 'package:test_sa/extensions/text_extensions.dart'; import 'package:test_sa/extensions/widget_extensions.dart'; import 'package:test_sa/new_views/app_style/app_color.dart'; -import 'package:test_sa/new_views/app_style/app_text_style.dart'; class ASoundPlayer extends StatefulWidget { final String audio; @@ -26,21 +24,73 @@ class _ASoundPlayerState extends State { bool _failedToLoad = false; late AudioPlayer _audioPlayer; - Widget audioPlayerButton(IconData icon, VoidCallback onpress) { + // Playback controls state + static const List _speedOptions = [1.0, 1.5, 2.0, 0.5]; + int _speedIndex = 0; + bool _muted = false; + + double get _playbackSpeed => _speedOptions[_speedIndex]; + + /// Circular play / pause / replay / download button. + Widget audioPlayerButton(IconData icon, VoidCallback onpress, {bool filled = true}) { return Container( - width: 48, - height: 48, + width: 44, + height: 44, decoration: ShapeDecoration( - color: AppColor.background(context), + color: filled ? AppColor.primary10 : AppColor.background(context), shape: RoundedRectangleBorder( - side: BorderSide(width: 1, color: AppColor.border2Color(context)), + side: filled ? BorderSide.none : BorderSide(width: 1, color: AppColor.border2Color(context)), borderRadius: BorderRadius.circular(32), ), ), - child: Icon(icon, size: 30, color: (AppColor.primary10)), + child: Icon(icon, size: 26, color: filled ? Colors.white : AppColor.primary10), ).onPress(onpress); } + /// Speed chip (cycles 1x -> 1.5x -> 2x -> 0.5x) matching the app theme. + Widget _speedChip() { + final label = _playbackSpeed == _playbackSpeed.roundToDouble() ? '${_playbackSpeed.toStringAsFixed(0)}x' : '${_playbackSpeed}x'; + return Container( + padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 6), + decoration: ShapeDecoration( + color: AppColor.primary10.withValues(alpha: 0.12), + shape: RoundedRectangleBorder(borderRadius: BorderRadius.circular(20)), + ), + child: Text( + label, + style: AppTextStyles.tinyFont.copyWith(color: AppColor.primary10, fontWeight: FontWeight.w700), + ), + ).onPress(() async { + _speedIndex = (_speedIndex + 1) % _speedOptions.length; + await _audioPlayer.setPlaybackRate(_playbackSpeed); + rebuild(); + }); + } + + /// Mute / un-mute toggle matching the app theme. + Widget _muteButton() { + return Container( + width: 34, + height: 34, + decoration: ShapeDecoration( + color: AppColor.background(context), + shape: RoundedRectangleBorder( + side: BorderSide(width: 1, color: AppColor.border2Color(context)), + borderRadius: BorderRadius.circular(20), + ), + ), + child: Icon( + _muted ? Icons.volume_off_rounded : Icons.volume_up_rounded, + size: 18, + color: _muted ? AppColor.lightTextColor(context) : AppColor.primary10, + ), + ).onPress(() async { + _muted = !_muted; + await _audioPlayer.setVolume(_muted ? 0.0 : 1.0); + rebuild(); + }); + } + Widget _getAudioButton() { switch (_audioPlayer.state) { case PlayerState.playing: @@ -167,40 +217,83 @@ class _ASoundPlayerState extends State { rebuild(); }); } - return Column( - crossAxisAlignment: CrossAxisAlignment.end, - children: [ - Row( - children: [ - _getAudioButton(), - SliderTheme( - data: SliderTheme.of(context).copyWith(trackHeight: 2.0, thumbShape: const RoundSliderThumbShape(enabledThumbRadius: 6.0), overlayShape: SliderComponentShape.noOverlay), - child: Slider( - value: _audioPosition?.inMilliseconds.toDouble() ?? 0.0, - min: 0, - max: _audioTime?.inMilliseconds.toDouble() ?? 60.0, - activeColor: AppColor.primary10, - inactiveColor: AppColor.border2Color(context), - onChangeStart: (value) { - _sliderMoving = true; - }, - onChanged: (value) { - _audioPosition = Duration(milliseconds: value.round()); - rebuild(); - }, - onChangeEnd: (value) { - _sliderMoving = false; - _audioPlayer.seek(Duration(milliseconds: value.round())); - rebuild(); - }, - ), - ).paddingOnly(start: 8, end: 8).expanded, - _failedToLoad - ? Text("Failed to load", style: AppTextStyles.tinyFont.copyWith(color: AppColor.textStyleColor(context))) - : Text("${format(_audioPosition ?? Duration.zero)}/${format(_audioTime ?? Duration.zero)}", style: AppTextStyles.tinyFont.copyWith(color: AppColor.textStyleColor(context))), - ], + return Container( + padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 10), + decoration: ShapeDecoration( + color: AppColor.background(context), + shape: RoundedRectangleBorder( + side: BorderSide(width: 1, color: AppColor.border2Color(context)), + borderRadius: BorderRadius.circular(16), ), - ], + ), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Row( + children: [ + _getAudioButton(), + Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + SliderTheme( + data: SliderTheme.of(context).copyWith( + trackHeight: 3.0, + thumbShape: const RoundSliderThumbShape(enabledThumbRadius: 6.0), + overlayShape: SliderComponentShape.noOverlay, + ), + child: Slider( + value: _sliderValue(), + min: 0, + max: _audioTime?.inMilliseconds.toDouble() ?? 60.0, + activeColor: AppColor.primary10, + inactiveColor: AppColor.border2Color(context), + onChangeStart: (value) { + _sliderMoving = true; + }, + onChanged: (value) { + _audioPosition = Duration(milliseconds: value.round()); + rebuild(); + }, + onChangeEnd: (value) { + _sliderMoving = false; + _audioPlayer.seek(Duration(milliseconds: value.round())); + rebuild(); + }, + ), + ), + Padding( + padding: const EdgeInsets.only(left: 8, right: 8), + child: Text( + _failedToLoad + ? "Failed to load" + : "${format(_audioPosition ?? Duration.zero)} / ${format(_audioTime ?? Duration.zero)}", + style: AppTextStyles.tinyFont.copyWith(color: AppColor.lightTextColor(context)), + ), + ), + ], + ).paddingOnly(start: 8, end: 8).expanded, + ], + ), + const SizedBox(height: 6), + Row( + mainAxisAlignment: MainAxisAlignment.end, + children: [ + _speedChip(), + const SizedBox(width: 10), + _muteButton(), + ], + ), + ], + ), ); } + + double _sliderValue() { + final pos = _audioPosition?.inMilliseconds.toDouble() ?? 0.0; + final max = _audioTime?.inMilliseconds.toDouble() ?? 60.0; + if (pos < 0) return 0; + if (pos > max) return max; + return pos; + } }