diff --git a/apps/ios/Sources/Capabilities/NodeCapabilityRouter.swift b/apps/ios/Sources/Capabilities/NodeCapabilityRouter.swift new file mode 100644 index 0000000000..6dbdd51eb8 --- /dev/null +++ b/apps/ios/Sources/Capabilities/NodeCapabilityRouter.swift @@ -0,0 +1,25 @@ +import Foundation +import OpenClawKit + +@MainActor +final class NodeCapabilityRouter { + enum RouterError: Error { + case unknownCommand + case handlerUnavailable + } + + typealias Handler = (BridgeInvokeRequest) async throws -> BridgeInvokeResponse + + private let handlers: [String: Handler] + + init(handlers: [String: Handler]) { + self.handlers = handlers + } + + func handle(_ request: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + guard let handler = handlers[request.command] else { + throw RouterError.unknownCommand + } + return try await handler(request) + } +} diff --git a/apps/ios/Sources/Gateway/GatewayConnectionController.swift b/apps/ios/Sources/Gateway/GatewayConnectionController.swift index 2a918bcfdb..1fc38c197c 100644 --- a/apps/ios/Sources/Gateway/GatewayConnectionController.swift +++ b/apps/ios/Sources/Gateway/GatewayConnectionController.swift @@ -214,8 +214,8 @@ final class GatewayConnectionController { guard let appModel else { return } let connectOptions = self.makeConnectOptions() - Task { [weak self] in - guard let self else { return } + Task { [weak appModel] in + guard let appModel else { return } await MainActor.run { appModel.gatewayStatusText = "Connecting…" } @@ -353,6 +353,7 @@ final class GatewayConnectionController { OpenClawCanvasA2UICommand.reset.rawValue, OpenClawScreenCommand.record.rawValue, OpenClawSystemCommand.notify.rawValue, + OpenClawChatCommand.push.rawValue, OpenClawTalkCommand.pttStart.rawValue, OpenClawTalkCommand.pttStop.rawValue, OpenClawTalkCommand.pttCancel.rawValue, diff --git a/apps/ios/Sources/Gateway/GatewayHealthMonitor.swift b/apps/ios/Sources/Gateway/GatewayHealthMonitor.swift new file mode 100644 index 0000000000..182df942c9 --- /dev/null +++ b/apps/ios/Sources/Gateway/GatewayHealthMonitor.swift @@ -0,0 +1,85 @@ +import Foundation +import OpenClawKit + +@MainActor +final class GatewayHealthMonitor { + struct Config: Sendable { + var intervalSeconds: Double + var timeoutSeconds: Double + var maxFailures: Int + } + + private let config: Config + private let sleep: @Sendable (UInt64) async -> Void + private var task: Task? + + init( + config: Config = Config(intervalSeconds: 15, timeoutSeconds: 5, maxFailures: 3), + sleep: @escaping @Sendable (UInt64) async -> Void = { nanoseconds in + try? await Task.sleep(nanoseconds: nanoseconds) + } + ) { + self.config = config + self.sleep = sleep + } + + func start( + check: @escaping @Sendable () async throws -> Bool, + onFailure: @escaping @Sendable (_ failureCount: Int) async -> Void) + { + self.stop() + let config = self.config + let sleep = self.sleep + self.task = Task { @MainActor in + var failures = 0 + while !Task.isCancelled { + let ok = await Self.runCheck(check: check, timeoutSeconds: config.timeoutSeconds) + if ok { + failures = 0 + } else { + failures += 1 + if failures >= max(1, config.maxFailures) { + await onFailure(failures) + failures = 0 + } + } + + if Task.isCancelled { break } + let interval = max(0.0, config.intervalSeconds) + let nanos = UInt64(interval * 1_000_000_000) + if nanos > 0 { + await sleep(nanos) + } else { + await Task.yield() + } + } + } + } + + func stop() { + self.task?.cancel() + self.task = nil + } + + private static func runCheck( + check: @escaping @Sendable () async throws -> Bool, + timeoutSeconds: Double) async -> Bool + { + let timeout = max(0.0, timeoutSeconds) + if timeout == 0 { + return (try? await check()) ?? false + } + do { + let timeoutError = NSError( + domain: "GatewayHealthMonitor", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "health check timed out"]) + return try await AsyncTimeout.withTimeout( + seconds: timeout, + onTimeout: { timeoutError }, + operation: check) + } catch { + return false + } + } +} diff --git a/apps/ios/Sources/Model/NodeAppModel.swift b/apps/ios/Sources/Model/NodeAppModel.swift index 12fe1be241..625c8e7104 100644 --- a/apps/ios/Sources/Model/NodeAppModel.swift +++ b/apps/ios/Sources/Model/NodeAppModel.swift @@ -1,3 +1,4 @@ +import OpenClawChatUI import OpenClawKit import Observation import SwiftUI @@ -61,6 +62,8 @@ final class NodeAppModel { private var gatewayTask: Task? private var voiceWakeSyncTask: Task? @ObservationIgnored private var cameraHUDDismissTask: Task? + @ObservationIgnored private var capabilityRouter: NodeCapabilityRouter + private let gatewayHealthMonitor = GatewayHealthMonitor() private let notificationCenter: NotificationCentering let voiceWake = VoiceWakeManager() let talkMode: TalkModeManager @@ -108,6 +111,8 @@ final class NodeAppModel { self.remindersService = remindersService self.motionService = motionService self.talkMode = talkMode + self.capabilityRouter = NodeCapabilityRouter(handlers: [:]) + self.capabilityRouter = self.buildCapabilityRouter() self.voiceWake.configure { [weak self] cmd in guard let self else { return } @@ -281,6 +286,7 @@ final class NodeAppModel { connectOptions: GatewayConnectOptions) { self.gatewayTask?.cancel() + self.gatewayHealthMonitor.stop() self.gatewayServerName = nil self.gatewayRemoteAddress = nil let id = gatewayStableID.trimmingCharacters(in: .whitespacesAndNewlines) @@ -325,6 +331,7 @@ final class NodeAppModel { } await self.refreshBrandingFromGateway() await self.startVoiceWakeSync() + await MainActor.run { self.startGatewayHealthMonitor() } await self.showA2UIOnConnectIfNeeded() }, onDisconnected: { [weak self] reason in @@ -337,6 +344,7 @@ final class NodeAppModel { self.showLocalCanvasOnDisconnect() self.gatewayStatusText = "Disconnected: \(reason)" } + await MainActor.run { self.stopGatewayHealthMonitor() } }, onInvoke: { [weak self] req in guard let self else { @@ -391,6 +399,7 @@ final class NodeAppModel { self.gatewayTask = nil self.voiceWakeSyncTask?.cancel() self.voiceWakeSyncTask = nil + self.gatewayHealthMonitor.stop() Task { await self.gateway.disconnect() } self.gatewayStatusText = "Offline" self.gatewayServerName = nil @@ -492,6 +501,27 @@ final class NodeAppModel { } } + private func startGatewayHealthMonitor() { + self.gatewayHealthMonitor.start( + check: { [weak self] in + guard let self else { return false } + do { + let data = try await self.gateway.request(method: "health", paramsJSON: nil, timeoutSeconds: 6) + return (try? JSONDecoder().decode(OpenClawGatewayHealthOK.self, from: data))?.ok ?? true + } catch { + return false + } + }, + onFailure: { [weak self] _ in + guard let self else { return } + await self.gateway.disconnect() + }) + } + + private func stopGatewayHealthMonitor() { + self.gatewayHealthMonitor.stop() + } + private func refreshWakeWordsFromGateway() async { do { let data = try await self.gateway.request(method: "voicewake.get", paramsJSON: "{}", timeoutSeconds: 8) @@ -597,54 +627,19 @@ final class NodeAppModel { } do { - switch command { - case OpenClawLocationCommand.get.rawValue: - return try await self.handleLocationInvoke(req) - case OpenClawCanvasCommand.present.rawValue, - OpenClawCanvasCommand.hide.rawValue, - OpenClawCanvasCommand.navigate.rawValue, - OpenClawCanvasCommand.evalJS.rawValue, - OpenClawCanvasCommand.snapshot.rawValue: - return try await self.handleCanvasInvoke(req) - case OpenClawCanvasA2UICommand.reset.rawValue, - OpenClawCanvasA2UICommand.push.rawValue, - OpenClawCanvasA2UICommand.pushJSONL.rawValue: - return try await self.handleCanvasA2UIInvoke(req) - case OpenClawCameraCommand.list.rawValue, - OpenClawCameraCommand.snap.rawValue, - OpenClawCameraCommand.clip.rawValue: - return try await self.handleCameraInvoke(req) - case OpenClawScreenCommand.record.rawValue: - return try await self.handleScreenRecordInvoke(req) - case OpenClawSystemCommand.notify.rawValue: - return try await self.handleSystemNotify(req) - case OpenClawDeviceCommand.status.rawValue, - OpenClawDeviceCommand.info.rawValue: - return try await self.handleDeviceInvoke(req) - case OpenClawPhotosCommand.latest.rawValue: - return try await self.handlePhotosInvoke(req) - case OpenClawContactsCommand.search.rawValue, - OpenClawContactsCommand.add.rawValue: - return try await self.handleContactsInvoke(req) - case OpenClawCalendarCommand.events.rawValue, - OpenClawCalendarCommand.add.rawValue: - return try await self.handleCalendarInvoke(req) - case OpenClawRemindersCommand.list.rawValue, - OpenClawRemindersCommand.add.rawValue: - return try await self.handleRemindersInvoke(req) - case OpenClawMotionCommand.activity.rawValue, - OpenClawMotionCommand.pedometer.rawValue: - return try await self.handleMotionInvoke(req) - case OpenClawTalkCommand.pttStart.rawValue, - OpenClawTalkCommand.pttStop.rawValue, - OpenClawTalkCommand.pttCancel.rawValue, - OpenClawTalkCommand.pttOnce.rawValue: - return try await self.handleTalkInvoke(req) - default: + return try await self.capabilityRouter.handle(req) + } catch let error as NodeCapabilityRouter.RouterError { + switch error { + case .unknownCommand: return BridgeInvokeResponse( id: req.id, ok: false, error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command")) + case .handlerUnavailable: + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError(code: .unavailable, message: "node handler unavailable")) } } catch { if command.hasPrefix("camera.") { @@ -983,6 +978,22 @@ final class NodeAppModel { let content = UNMutableNotificationContent() content.title = title content.body = body + if #available(iOS 15.0, *) { + switch params.priority ?? .active { + case .passive: + content.interruptionLevel = .passive + case .timeSensitive: + content.interruptionLevel = .timeSensitive + case .active: + content.interruptionLevel = .active + } + } + let soundValue = params.sound?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if let soundValue, ["none", "silent", "off", "false", "0"].contains(soundValue) { + content.sound = nil + } else { + content.sound = .default + } let request = UNNotificationRequest( identifier: UUID().uuidString, content: content, @@ -998,6 +1009,51 @@ final class NodeAppModel { return BridgeInvokeResponse(id: req.id, ok: true) } + private func handleChatPushInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let params = try Self.decodeParams(OpenClawChatPushParams.self, from: req.paramsJSON) + let text = params.text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: empty chat.push text")) + } + + let finalStatus = await self.requestNotificationAuthorizationIfNeeded() + let messageId = UUID().uuidString + if finalStatus == .authorized || finalStatus == .provisional || finalStatus == .ephemeral { + let addResult = await self.runNotificationCall(timeoutSeconds: 2.0) { [notificationCenter] in + let content = UNMutableNotificationContent() + content.title = "OpenClaw" + content.body = text + content.sound = .default + content.userInfo = ["messageId": messageId] + let request = UNNotificationRequest( + identifier: messageId, + content: content, + trigger: nil) + try await notificationCenter.add(request) + } + if case let .failure(error) = addResult { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError(code: .unavailable, message: "NOTIFICATION_FAILED: \(error.message)")) + } + } + + if params.speak ?? true { + let toSpeak = text + Task { @MainActor in + try? await TalkSystemSpeechSynthesizer.shared.speak(text: toSpeak) + } + } + + let payload = OpenClawChatPushPayload(messageId: messageId) + let json = try Self.encodePayload(payload) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json) + } + private func requestNotificationAuthorizationIfNeeded() async -> NotificationAuthorizationStatus { let status = await self.notificationAuthorizationStatus() guard status == .notDetermined else { return status } @@ -1203,6 +1259,123 @@ final class NodeAppModel { } private extension NodeAppModel { + // Central registry for node invoke routing to keep commands in one place. + func buildCapabilityRouter() -> NodeCapabilityRouter { + var handlers: [String: NodeCapabilityRouter.Handler] = [:] + + func register(_ commands: [String], handler: @escaping NodeCapabilityRouter.Handler) { + for command in commands { + handlers[command] = handler + } + } + + register([OpenClawLocationCommand.get.rawValue]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handleLocationInvoke(req) + } + + register([ + OpenClawCanvasCommand.present.rawValue, + OpenClawCanvasCommand.hide.rawValue, + OpenClawCanvasCommand.navigate.rawValue, + OpenClawCanvasCommand.evalJS.rawValue, + OpenClawCanvasCommand.snapshot.rawValue, + ]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handleCanvasInvoke(req) + } + + register([ + OpenClawCanvasA2UICommand.reset.rawValue, + OpenClawCanvasA2UICommand.push.rawValue, + OpenClawCanvasA2UICommand.pushJSONL.rawValue, + ]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handleCanvasA2UIInvoke(req) + } + + register([ + OpenClawCameraCommand.list.rawValue, + OpenClawCameraCommand.snap.rawValue, + OpenClawCameraCommand.clip.rawValue, + ]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handleCameraInvoke(req) + } + + register([OpenClawScreenCommand.record.rawValue]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handleScreenRecordInvoke(req) + } + + register([OpenClawSystemCommand.notify.rawValue]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handleSystemNotify(req) + } + + register([OpenClawChatCommand.push.rawValue]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handleChatPushInvoke(req) + } + + register([ + OpenClawDeviceCommand.status.rawValue, + OpenClawDeviceCommand.info.rawValue, + ]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handleDeviceInvoke(req) + } + + register([OpenClawPhotosCommand.latest.rawValue]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handlePhotosInvoke(req) + } + + register([ + OpenClawContactsCommand.search.rawValue, + OpenClawContactsCommand.add.rawValue, + ]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handleContactsInvoke(req) + } + + register([ + OpenClawCalendarCommand.events.rawValue, + OpenClawCalendarCommand.add.rawValue, + ]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handleCalendarInvoke(req) + } + + register([ + OpenClawRemindersCommand.list.rawValue, + OpenClawRemindersCommand.add.rawValue, + ]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handleRemindersInvoke(req) + } + + register([ + OpenClawMotionCommand.activity.rawValue, + OpenClawMotionCommand.pedometer.rawValue, + ]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handleMotionInvoke(req) + } + + register([ + OpenClawTalkCommand.pttStart.rawValue, + OpenClawTalkCommand.pttStop.rawValue, + OpenClawTalkCommand.pttCancel.rawValue, + OpenClawTalkCommand.pttOnce.rawValue, + ]) { [weak self] req in + guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable } + return try await self.handleTalkInvoke(req) + } + + return NodeCapabilityRouter(handlers: handlers) + } + func locationMode() -> OpenClawLocationMode { let raw = UserDefaults.standard.string(forKey: "location.enabledMode") ?? "off" return OpenClawLocationMode(rawValue: raw) ?? .off diff --git a/apps/ios/Sources/Voice/TalkModeManager.swift b/apps/ios/Sources/Voice/TalkModeManager.swift index 6d7c478128..d7038f67b6 100644 --- a/apps/ios/Sources/Voice/TalkModeManager.swift +++ b/apps/ios/Sources/Voice/TalkModeManager.swift @@ -1,4 +1,5 @@ import AVFAudio +import OpenClawChatUI import OpenClawKit import OpenClawProtocol import Foundation @@ -65,6 +66,14 @@ final class TalkModeManager: NSObject { private let silenceWindow: TimeInterval = 0.7 private var chatSubscribedSessionKeys = Set() + private var incrementalSpeechQueue: [String] = [] + private var incrementalSpeechTask: Task? + private var incrementalSpeechActive = false + private var incrementalSpeechUsed = false + private var incrementalSpeechLanguage: String? + private var incrementalSpeechBuffer = IncrementalSpeechBuffer() + private var incrementalSpeechContext: IncrementalSpeechContext? + private var incrementalSpeechDirective: TalkDirective? private let logger = Logger(subsystem: "bot.molt", category: "TalkMode") @@ -456,6 +465,14 @@ final class TalkModeManager: NSObject { } if isFinal { self.lastTranscript = trimmed + guard !trimmed.isEmpty else { return } + if self.captureMode == .pushToTalk, self.pttAutoStopEnabled, self.isPushToTalkActive { + _ = await self.endPushToTalk() + return + } + if self.captureMode == .continuous, !self.isSpeaking { + await self.processTranscript(trimmed, restartAfter: true) + } } } @@ -539,6 +556,15 @@ final class TalkModeManager: NSObject { "chat.send start sessionKey=\(sessionKey, privacy: .public) chars=\(prompt.count, privacy: .public)") let runId = try await self.sendChat(prompt, gateway: gateway) self.logger.info("chat.send ok runId=\(runId, privacy: .public)") + let shouldIncremental = self.shouldUseIncrementalTTS() + var streamingTask: Task? + if shouldIncremental { + self.resetIncrementalSpeech() + streamingTask = Task { @MainActor [weak self] in + guard let self else { return } + await self.streamAssistant(runId: runId, gateway: gateway) + } + } let completion = await self.waitForChatCompletion(runId: runId, gateway: gateway, timeoutSeconds: 120) if completion == .timeout { self.logger.warning( @@ -546,27 +572,44 @@ final class TalkModeManager: NSObject { } else if completion == .aborted { self.statusText = "Aborted" self.logger.warning("chat completion aborted runId=\(runId, privacy: .public)") + streamingTask?.cancel() + await self.finishIncrementalSpeech() await self.start() return } else if completion == .error { self.statusText = "Chat error" self.logger.warning("chat completion error runId=\(runId, privacy: .public)") + streamingTask?.cancel() + await self.finishIncrementalSpeech() await self.start() return } - guard let assistantText = try await self.waitForAssistantText( + var assistantText = try await self.waitForAssistantText( gateway: gateway, since: startedAt, timeoutSeconds: completion == .final ? 12 : 25) - else { + if assistantText == nil, shouldIncremental { + let fallback = self.incrementalSpeechBuffer.latestText + if !fallback.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + assistantText = fallback + } + } + guard let assistantText else { self.statusText = "No reply" self.logger.warning("assistant text timeout runId=\(runId, privacy: .public)") + streamingTask?.cancel() + await self.finishIncrementalSpeech() await self.start() return } self.logger.info("assistant text ok chars=\(assistantText.count, privacy: .public)") - await self.playAssistant(text: assistantText) + streamingTask?.cancel() + if shouldIncremental { + await self.handleIncrementalAssistantFinal(text: assistantText) + } else { + await self.playAssistant(text: assistantText) + } } catch { self.statusText = "Talk failed: \(error.localizedDescription)" self.logger.error("finalize failed: \(error.localizedDescription, privacy: .public)") @@ -720,24 +763,7 @@ final class TalkModeManager: NSObject { let directive = parsed.directive let cleaned = parsed.stripped.trimmingCharacters(in: .whitespacesAndNewlines) guard !cleaned.isEmpty else { return } - - let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines) - let resolvedVoice = self.resolveVoiceAlias(requestedVoice) - if requestedVoice?.isEmpty == false, resolvedVoice == nil { - self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)") - } - if let voice = resolvedVoice { - if directive?.once != true { - self.currentVoiceId = voice - self.voiceOverrideActive = true - } - } - if let model = directive?.modelId { - if directive?.once != true { - self.currentModelId = model - self.modelOverrideActive = true - } - } + self.applyDirective(directive) self.statusText = "Generating voice…" self.isSpeaking = true @@ -746,6 +772,11 @@ final class TalkModeManager: NSObject { do { let started = Date() let language = ElevenLabsTTSClient.validatedLanguage(directive?.language) + let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines) + let resolvedVoice = self.resolveVoiceAlias(requestedVoice) + if requestedVoice?.isEmpty == false, resolvedVoice == nil { + self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)") + } let resolvedKey = (self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ?? @@ -875,6 +906,7 @@ final class TalkModeManager: NSObject { ? self.mp3Player.stop() : self.pcmPlayer.stop() TalkSystemSpeechSynthesizer.shared.stop() + self.cancelIncrementalSpeech() self.isSpeaking = false } @@ -887,6 +919,268 @@ final class TalkModeManager: NSObject { return true } + private func shouldUseIncrementalTTS() -> Bool { + true + } + + private func applyDirective(_ directive: TalkDirective?) { + let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines) + let resolvedVoice = self.resolveVoiceAlias(requestedVoice) + if requestedVoice?.isEmpty == false, resolvedVoice == nil { + self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)") + } + if let voice = resolvedVoice { + if directive?.once != true { + self.currentVoiceId = voice + self.voiceOverrideActive = true + } + } + if let model = directive?.modelId { + if directive?.once != true { + self.currentModelId = model + self.modelOverrideActive = true + } + } + } + + private func resetIncrementalSpeech() { + self.incrementalSpeechQueue.removeAll() + self.incrementalSpeechTask?.cancel() + self.incrementalSpeechTask = nil + self.incrementalSpeechActive = true + self.incrementalSpeechUsed = false + self.incrementalSpeechLanguage = nil + self.incrementalSpeechBuffer = IncrementalSpeechBuffer() + self.incrementalSpeechContext = nil + self.incrementalSpeechDirective = nil + } + + private func cancelIncrementalSpeech() { + self.incrementalSpeechQueue.removeAll() + self.incrementalSpeechTask?.cancel() + self.incrementalSpeechTask = nil + self.incrementalSpeechActive = false + self.incrementalSpeechContext = nil + self.incrementalSpeechDirective = nil + } + + private func enqueueIncrementalSpeech(_ text: String) { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + self.incrementalSpeechQueue.append(trimmed) + self.incrementalSpeechUsed = true + if self.incrementalSpeechTask == nil { + self.startIncrementalSpeechTask() + } + } + + private func startIncrementalSpeechTask() { + if self.interruptOnSpeech { + do { + try self.startRecognition() + } catch { + self.logger.warning( + "startRecognition during incremental speak failed: \(error.localizedDescription, privacy: .public)") + } + } + + self.incrementalSpeechTask = Task { @MainActor [weak self] in + guard let self else { return } + while !Task.isCancelled { + guard !self.incrementalSpeechQueue.isEmpty else { break } + let segment = self.incrementalSpeechQueue.removeFirst() + self.statusText = "Speaking…" + self.isSpeaking = true + self.lastSpokenText = segment + await self.speakIncrementalSegment(segment) + } + self.isSpeaking = false + self.stopRecognition() + self.incrementalSpeechTask = nil + } + } + + private func finishIncrementalSpeech() async { + guard self.incrementalSpeechActive else { return } + let leftover = self.incrementalSpeechBuffer.flush() + if let leftover { + self.enqueueIncrementalSpeech(leftover) + } + if let task = self.incrementalSpeechTask { + _ = await task.result + } + self.incrementalSpeechActive = false + } + + private func handleIncrementalAssistantFinal(text: String) async { + let parsed = TalkDirectiveParser.parse(text) + self.applyDirective(parsed.directive) + if let lang = parsed.directive?.language { + self.incrementalSpeechLanguage = ElevenLabsTTSClient.validatedLanguage(lang) + } + await self.updateIncrementalContextIfNeeded() + let segments = self.incrementalSpeechBuffer.ingest(text: text, isFinal: true) + for segment in segments { + self.enqueueIncrementalSpeech(segment) + } + await self.finishIncrementalSpeech() + if !self.incrementalSpeechUsed { + await self.playAssistant(text: text) + } + } + + private func streamAssistant(runId: String, gateway: GatewayNodeSession) async { + let stream = await gateway.subscribeServerEvents(bufferingNewest: 200) + for await evt in stream { + if Task.isCancelled { return } + guard evt.event == "agent", let payload = evt.payload else { continue } + guard let agentEvent = try? GatewayPayloadDecoding.decode(payload, as: OpenClawAgentEventPayload.self) else { + continue + } + guard agentEvent.runId == runId, agentEvent.stream == "assistant" else { continue } + guard let text = agentEvent.data["text"]?.value as? String else { continue } + let segments = self.incrementalSpeechBuffer.ingest(text: text, isFinal: false) + if let lang = self.incrementalSpeechBuffer.directive?.language { + self.incrementalSpeechLanguage = ElevenLabsTTSClient.validatedLanguage(lang) + } + await self.updateIncrementalContextIfNeeded() + for segment in segments { + self.enqueueIncrementalSpeech(segment) + } + } + } + + private func updateIncrementalContextIfNeeded() async { + let directive = self.incrementalSpeechBuffer.directive + if let existing = self.incrementalSpeechContext, directive == self.incrementalSpeechDirective { + if existing.language != self.incrementalSpeechLanguage { + self.incrementalSpeechContext = IncrementalSpeechContext( + apiKey: existing.apiKey, + voiceId: existing.voiceId, + modelId: existing.modelId, + outputFormat: existing.outputFormat, + language: self.incrementalSpeechLanguage, + directive: existing.directive, + canUseElevenLabs: existing.canUseElevenLabs) + } + return + } + let context = await self.buildIncrementalSpeechContext(directive: directive) + self.incrementalSpeechContext = context + self.incrementalSpeechDirective = directive + } + + private func buildIncrementalSpeechContext(directive: TalkDirective?) async -> IncrementalSpeechContext { + let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines) + let resolvedVoice = self.resolveVoiceAlias(requestedVoice) + if requestedVoice?.isEmpty == false, resolvedVoice == nil { + self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)") + } + let preferredVoice = resolvedVoice ?? self.currentVoiceId ?? self.defaultVoiceId + let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId + let desiredOutputFormat = (directive?.outputFormat ?? self.defaultOutputFormat)? + .trimmingCharacters(in: .whitespacesAndNewlines) + let requestedOutputFormat = (desiredOutputFormat?.isEmpty == false) ? desiredOutputFormat : nil + let outputFormat = ElevenLabsTTSClient.validatedOutputFormat(requestedOutputFormat ?? "pcm_44100") + if outputFormat == nil, let requestedOutputFormat { + self.logger.warning( + "talk output_format unsupported for local playback: \(requestedOutputFormat, privacy: .public)") + } + + let resolvedKey = + (self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ?? + ProcessInfo.processInfo.environment["ELEVENLABS_API_KEY"] + let apiKey = resolvedKey?.trimmingCharacters(in: .whitespacesAndNewlines) + let voiceId: String? = if let apiKey, !apiKey.isEmpty { + await self.resolveVoiceId(preferred: preferredVoice, apiKey: apiKey) + } else { + nil + } + let canUseElevenLabs = (voiceId?.isEmpty == false) && (apiKey?.isEmpty == false) + return IncrementalSpeechContext( + apiKey: apiKey, + voiceId: voiceId, + modelId: modelId, + outputFormat: outputFormat, + language: self.incrementalSpeechLanguage, + directive: directive, + canUseElevenLabs: canUseElevenLabs) + } + + private func speakIncrementalSegment(_ text: String) async { + await self.updateIncrementalContextIfNeeded() + guard let context = self.incrementalSpeechContext else { + try? await TalkSystemSpeechSynthesizer.shared.speak( + text: text, + language: self.incrementalSpeechLanguage) + return + } + + if context.canUseElevenLabs, let apiKey = context.apiKey, let voiceId = context.voiceId { + let request = ElevenLabsTTSRequest( + text: text, + modelId: context.modelId, + outputFormat: context.outputFormat, + speed: TalkTTSValidation.resolveSpeed( + speed: context.directive?.speed, + rateWPM: context.directive?.rateWPM), + stability: TalkTTSValidation.validatedStability( + context.directive?.stability, + modelId: context.modelId), + similarity: TalkTTSValidation.validatedUnit(context.directive?.similarity), + style: TalkTTSValidation.validatedUnit(context.directive?.style), + speakerBoost: context.directive?.speakerBoost, + seed: TalkTTSValidation.validatedSeed(context.directive?.seed), + normalize: ElevenLabsTTSClient.validatedNormalize(context.directive?.normalize), + language: context.language, + latencyTier: TalkTTSValidation.validatedLatencyTier(context.directive?.latencyTier)) + let client = ElevenLabsTTSClient(apiKey: apiKey) + let stream = client.streamSynthesize(voiceId: voiceId, request: request) + let sampleRate = TalkTTSValidation.pcmSampleRate(from: context.outputFormat) + let result: StreamingPlaybackResult + if let sampleRate { + self.lastPlaybackWasPCM = true + var playback = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate) + if !playback.finished, playback.interruptedAt == nil { + self.logger.warning("pcm playback failed; retrying mp3") + self.lastPlaybackWasPCM = false + let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100") + let mp3Stream = client.streamSynthesize( + voiceId: voiceId, + request: ElevenLabsTTSRequest( + text: text, + modelId: context.modelId, + outputFormat: mp3Format, + speed: TalkTTSValidation.resolveSpeed( + speed: context.directive?.speed, + rateWPM: context.directive?.rateWPM), + stability: TalkTTSValidation.validatedStability( + context.directive?.stability, + modelId: context.modelId), + similarity: TalkTTSValidation.validatedUnit(context.directive?.similarity), + style: TalkTTSValidation.validatedUnit(context.directive?.style), + speakerBoost: context.directive?.speakerBoost, + seed: TalkTTSValidation.validatedSeed(context.directive?.seed), + normalize: ElevenLabsTTSClient.validatedNormalize(context.directive?.normalize), + language: context.language, + latencyTier: TalkTTSValidation.validatedLatencyTier(context.directive?.latencyTier))) + playback = await self.mp3Player.play(stream: mp3Stream) + } + result = playback + } else { + self.lastPlaybackWasPCM = false + result = await self.mp3Player.play(stream: stream) + } + if !result.finished, let interruptedAt = result.interruptedAt { + self.lastInterruptedAtSeconds = interruptedAt + } + } else { + try? await TalkSystemSpeechSynthesizer.shared.speak( + text: text, + language: self.incrementalSpeechLanguage) + } + } + private func resolveVoiceAlias(_ value: String?) -> String? { let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return nil } @@ -1010,6 +1304,121 @@ final class TalkModeManager: NSObject { } } +private struct IncrementalSpeechBuffer { + private(set) var latestText: String = "" + private(set) var directive: TalkDirective? + private var spokenOffset: Int = 0 + private var inCodeBlock = false + private var directiveParsed = false + + mutating func ingest(text: String, isFinal: Bool) -> [String] { + let normalized = text.replacingOccurrences(of: "\r\n", with: "\n") + guard let usable = self.stripDirectiveIfReady(from: normalized) else { return [] } + self.updateText(usable) + return self.extractSegments(isFinal: isFinal) + } + + mutating func flush() -> String? { + guard !self.latestText.isEmpty else { return nil } + let segments = self.extractSegments(isFinal: true) + return segments.first + } + + private mutating func stripDirectiveIfReady(from text: String) -> String? { + guard !self.directiveParsed else { return text } + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + if trimmed.hasPrefix("{") { + guard let newlineRange = text.range(of: "\n") else { return nil } + let firstLine = text[.. self.latestText.count { + self.spokenOffset = self.latestText.count + } + } + + private mutating func extractSegments(isFinal: Bool) -> [String] { + let chars = Array(self.latestText) + guard self.spokenOffset < chars.count else { return [] } + var idx = self.spokenOffset + var lastBoundary: Int? + var inCodeBlock = self.inCodeBlock + var buffer = "" + var bufferAtBoundary = "" + var inCodeBlockAtBoundary = inCodeBlock + + while idx < chars.count { + if idx + 2 < chars.count, + chars[idx] == "`", + chars[idx + 1] == "`", + chars[idx + 2] == "`" + { + inCodeBlock.toggle() + idx += 3 + continue + } + + if !inCodeBlock { + buffer.append(chars[idx]) + if Self.isBoundary(chars[idx]) { + lastBoundary = idx + 1 + bufferAtBoundary = buffer + inCodeBlockAtBoundary = inCodeBlock + } + } + + idx += 1 + } + + if let boundary = lastBoundary { + self.spokenOffset = boundary + self.inCodeBlock = inCodeBlockAtBoundary + let trimmed = bufferAtBoundary.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? [] : [trimmed] + } + + guard isFinal else { return [] } + self.spokenOffset = chars.count + self.inCodeBlock = inCodeBlock + let trimmed = buffer.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? [] : [trimmed] + } + + private static func isBoundary(_ ch: Character) -> Bool { + ch == "." || ch == "!" || ch == "?" || ch == "\n" + } +} + +private struct IncrementalSpeechContext { + let apiKey: String? + let voiceId: String? + let modelId: String? + let outputFormat: String? + let language: String? + let directive: TalkDirective? + let canUseElevenLabs: Bool +} + #if DEBUG extension TalkModeManager { func _test_seedTranscript(_ transcript: String) { @@ -1017,6 +1426,10 @@ extension TalkModeManager { self.lastHeard = Date() } + func _test_handleTranscript(_ transcript: String, isFinal: Bool) async { + await self.handleTranscript(transcript: transcript, isFinal: isFinal) + } + func _test_backdateLastHeard(seconds: TimeInterval) { self.lastHeard = Date().addingTimeInterval(-seconds) } @@ -1024,5 +1437,13 @@ extension TalkModeManager { func _test_runSilenceCheck() async { await self.checkSilence() } + + func _test_incrementalReset() { + self.incrementalSpeechBuffer = IncrementalSpeechBuffer() + } + + func _test_incrementalIngest(_ text: String, isFinal: Bool) -> [String] { + self.incrementalSpeechBuffer.ingest(text: text, isFinal: isFinal) + } } #endif diff --git a/apps/ios/SwiftSources.input.xcfilelist b/apps/ios/SwiftSources.input.xcfilelist index cdfb272469..4b94dbe3c8 100644 --- a/apps/ios/SwiftSources.input.xcfilelist +++ b/apps/ios/SwiftSources.input.xcfilelist @@ -2,7 +2,9 @@ Sources/Gateway/GatewayConnectionController.swift Sources/Gateway/GatewayDiscoveryDebugLogView.swift Sources/Gateway/GatewayDiscoveryModel.swift Sources/Gateway/GatewaySettingsStore.swift +Sources/Gateway/GatewayHealthMonitor.swift Sources/Gateway/KeychainStore.swift +Sources/Capabilities/NodeCapabilityRouter.swift Sources/Camera/CameraController.swift Sources/Chat/ChatSheet.swift Sources/Chat/IOSGatewayChatTransport.swift @@ -58,6 +60,7 @@ Sources/Voice/VoiceWakePreferences.swift ../shared/OpenClawKit/Sources/OpenClawKit/CanvasCommandParams.swift ../shared/OpenClawKit/Sources/OpenClawKit/CanvasCommands.swift ../shared/OpenClawKit/Sources/OpenClawKit/Capabilities.swift +../shared/OpenClawKit/Sources/OpenClawKit/ChatCommands.swift ../shared/OpenClawKit/Sources/OpenClawKit/ContactsCommands.swift ../shared/OpenClawKit/Sources/OpenClawKit/DeviceCommands.swift ../shared/OpenClawKit/Sources/OpenClawKit/OpenClawKitResources.swift diff --git a/apps/ios/Tests/GatewayConnectionControllerTests.swift b/apps/ios/Tests/GatewayConnectionControllerTests.swift index f21468b198..6cdb37decc 100644 --- a/apps/ios/Tests/GatewayConnectionControllerTests.swift +++ b/apps/ios/Tests/GatewayConnectionControllerTests.swift @@ -92,6 +92,7 @@ private func withUserDefaults(_ updates: [String: Any?], _ body: () throws -> let commands = Set(controller._test_currentCommands()) #expect(commands.contains(OpenClawSystemCommand.notify.rawValue)) + #expect(commands.contains(OpenClawChatCommand.push.rawValue)) #expect(!commands.contains(OpenClawSystemCommand.run.rawValue)) #expect(!commands.contains(OpenClawSystemCommand.which.rawValue)) #expect(!commands.contains(OpenClawSystemCommand.execApprovalsGet.rawValue)) diff --git a/apps/ios/Tests/GatewayHealthMonitorTests.swift b/apps/ios/Tests/GatewayHealthMonitorTests.swift new file mode 100644 index 0000000000..38b46edc51 --- /dev/null +++ b/apps/ios/Tests/GatewayHealthMonitorTests.swift @@ -0,0 +1,60 @@ +import Foundation +import Testing +@testable import OpenClaw + +private actor Counter { + private var value = 0 + + func increment() { + value += 1 + } + + func get() -> Int { + value + } + + func set(_ newValue: Int) { + value = newValue + } +} + +@Suite struct GatewayHealthMonitorTests { + @Test @MainActor func triggersFailureAfterThreshold() async { + let failureCount = Counter() + let monitor = GatewayHealthMonitor( + config: .init(intervalSeconds: 0.001, timeoutSeconds: 0.0, maxFailures: 2)) + + monitor.start( + check: { false }, + onFailure: { _ in + await failureCount.increment() + await monitor.stop() + }) + + try? await Task.sleep(nanoseconds: 60_000_000) + #expect(await failureCount.get() == 1) + } + + @Test @MainActor func resetsFailuresAfterSuccess() async { + let failureCount = Counter() + let calls = Counter() + let monitor = GatewayHealthMonitor( + config: .init(intervalSeconds: 0.001, timeoutSeconds: 0.0, maxFailures: 2)) + + monitor.start( + check: { + await calls.increment() + let callCount = await calls.get() + if callCount >= 6 { + await monitor.stop() + } + return callCount % 2 == 0 + }, + onFailure: { _ in + await failureCount.increment() + }) + + try? await Task.sleep(nanoseconds: 60_000_000) + #expect(await failureCount.get() == 0) + } +} diff --git a/apps/ios/Tests/NodeAppModelInvokeTests.swift b/apps/ios/Tests/NodeAppModelInvokeTests.swift index 1aa665ea4a..512faf22cc 100644 --- a/apps/ios/Tests/NodeAppModelInvokeTests.swift +++ b/apps/ios/Tests/NodeAppModelInvokeTests.swift @@ -448,6 +448,91 @@ private func decodePayload(_ json: String?, as type: T.Type) throw #expect(request.content.body == "World") } + @Test @MainActor func handleInvokeChatPushCreatesNotification() async throws { + let notifier = TestNotificationCenter(status: .authorized) + let deviceStatus = TestDeviceStatusService( + statusPayload: OpenClawDeviceStatusPayload( + battery: OpenClawBatteryStatusPayload(level: 0.5, state: .charging, lowPowerModeEnabled: false), + thermal: OpenClawThermalStatusPayload(state: .nominal), + storage: OpenClawStorageStatusPayload(totalBytes: 100, freeBytes: 50, usedBytes: 50), + network: OpenClawNetworkStatusPayload( + status: .satisfied, + isExpensive: false, + isConstrained: false, + interfaces: [.wifi]), + uptimeSeconds: 10), + infoPayload: OpenClawDeviceInfoPayload( + deviceName: "Test", + modelIdentifier: "Test1,1", + systemName: "iOS", + systemVersion: "1.0", + appVersion: "dev", + appBuild: "0", + locale: "en-US")) + let emptyContact = OpenClawContactPayload( + identifier: "c0", + displayName: "", + givenName: "", + familyName: "", + organizationName: "", + phoneNumbers: [], + emails: []) + let emptyEvent = OpenClawCalendarEventPayload( + identifier: "e0", + title: "Test", + startISO: "2024-01-01T00:00:00Z", + endISO: "2024-01-01T00:30:00Z", + isAllDay: false, + location: nil, + calendarTitle: nil) + let emptyReminder = OpenClawReminderPayload( + identifier: "r0", + title: "Test", + dueISO: nil, + completed: false, + listName: nil) + let appModel = makeTestAppModel( + notificationCenter: notifier, + deviceStatusService: deviceStatus, + photosService: TestPhotosService(payload: OpenClawPhotosLatestPayload(photos: [])), + contactsService: TestContactsService( + searchPayload: OpenClawContactsSearchPayload(contacts: []), + addPayload: OpenClawContactsAddPayload(contact: emptyContact)), + calendarService: TestCalendarService( + eventsPayload: OpenClawCalendarEventsPayload(events: []), + addPayload: OpenClawCalendarAddPayload(event: emptyEvent)), + remindersService: TestRemindersService( + listPayload: OpenClawRemindersListPayload(reminders: []), + addPayload: OpenClawRemindersAddPayload(reminder: emptyReminder)), + motionService: TestMotionService( + activityPayload: OpenClawMotionActivityPayload(activities: []), + pedometerPayload: OpenClawPedometerPayload( + startISO: "2024-01-01T00:00:00Z", + endISO: "2024-01-01T01:00:00Z", + steps: nil, + distanceMeters: nil, + floorsAscended: nil, + floorsDescended: nil))) + + let params = OpenClawChatPushParams(text: "Ping", speak: false) + let data = try JSONEncoder().encode(params) + let json = String(decoding: data, as: UTF8.self) + let req = BridgeInvokeRequest( + id: "chat-push", + command: OpenClawChatCommand.push.rawValue, + paramsJSON: json) + let res = await appModel._test_handleInvoke(req) + #expect(res.ok == true) + #expect(notifier.addedRequests.count == 1) + let request = try #require(notifier.addedRequests.first) + #expect(request.content.title == "OpenClaw") + #expect(request.content.body == "Ping") + let payloadJSON = try #require(res.payloadJSON) + let decoded = try JSONDecoder().decode(OpenClawChatPushPayload.self, from: Data(payloadJSON.utf8)) + #expect((decoded.messageId ?? "").isEmpty == false) + #expect(request.identifier == decoded.messageId) + } + @Test @MainActor func handleInvokeDeviceAndDataCommandsReturnPayloads() async throws { let deviceStatusPayload = OpenClawDeviceStatusPayload( battery: OpenClawBatteryStatusPayload(level: 0.25, state: .unplugged, lowPowerModeEnabled: false), @@ -723,6 +808,28 @@ private func decodePayload(_ json: String?, as type: T.Type) throw #expect(oncePayload.status == "offline") } + @Test @MainActor func handleInvokePushToTalkOnceStopsOnFinalTranscript() async throws { + let talkMode = TalkModeManager(allowSimulatorCapture: true) + talkMode.updateGatewayConnected(false) + let appModel = makeTalkTestAppModel(talkMode: talkMode) + + let onceReq = BridgeInvokeRequest(id: "ptt-once-final", command: OpenClawTalkCommand.pttOnce.rawValue) + let onceTask = Task { await appModel._test_handleInvoke(onceReq) } + + for _ in 0..<5 where !talkMode.isPushToTalkActive { + await Task.yield() + } + #expect(talkMode.isPushToTalkActive == true) + + await talkMode._test_handleTranscript("Hello final", isFinal: true) + + let onceRes = await onceTask.value + #expect(onceRes.ok == true) + let oncePayload = try decodePayload(onceRes.payloadJSON, as: OpenClawTalkPTTStopPayload.self) + #expect(oncePayload.transcript == "Hello final") + #expect(oncePayload.status == "offline") + } + @Test @MainActor func handleDeepLinkSetsErrorWhenNotConnected() async { let appModel = NodeAppModel() let url = URL(string: "openclaw://agent?message=hello")! diff --git a/apps/ios/Tests/TalkModeIncrementalTests.swift b/apps/ios/Tests/TalkModeIncrementalTests.swift new file mode 100644 index 0000000000..9bd17f07d5 --- /dev/null +++ b/apps/ios/Tests/TalkModeIncrementalTests.swift @@ -0,0 +1,33 @@ +import Foundation +import Testing +@testable import OpenClaw + +@Suite struct TalkModeIncrementalTests { + @Test @MainActor func incrementalSpeechSplitsOnBoundary() { + let talkMode = TalkModeManager(allowSimulatorCapture: true) + talkMode._test_incrementalReset() + let segments = talkMode._test_incrementalIngest("Hello world. Next", isFinal: false) + #expect(segments.count == 1) + #expect(segments.first == "Hello world.") + } + + @Test @MainActor func incrementalSpeechSkipsDirectiveLine() { + let talkMode = TalkModeManager(allowSimulatorCapture: true) + talkMode._test_incrementalReset() + let segments = talkMode._test_incrementalIngest("{\"voice\":\"abc\"}\nHello.", isFinal: false) + #expect(segments.count == 1) + #expect(segments.first == "Hello.") + } + + @Test @MainActor func incrementalSpeechIgnoresCodeBlocks() { + let talkMode = TalkModeManager(allowSimulatorCapture: true) + talkMode._test_incrementalReset() + let text = "Here is code:\n```js\nx=1\n```\nDone." + let segments = talkMode._test_incrementalIngest(text, isFinal: true) + #expect(segments.count == 1) + let value = segments.first ?? "" + #expect(value.contains("x=1") == false) + #expect(value.contains("Here is code") == true) + #expect(value.contains("Done.") == true) + } +} diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/ChatCommands.swift b/apps/shared/OpenClawKit/Sources/OpenClawKit/ChatCommands.swift new file mode 100644 index 0000000000..98bac6205d --- /dev/null +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/ChatCommands.swift @@ -0,0 +1,23 @@ +import Foundation + +public enum OpenClawChatCommand: String, Codable, Sendable { + case push = "chat.push" +} + +public struct OpenClawChatPushParams: Codable, Sendable, Equatable { + public var text: String + public var speak: Bool? + + public init(text: String, speak: Bool? = nil) { + self.text = text + self.speak = speak + } +} + +public struct OpenClawChatPushPayload: Codable, Sendable, Equatable { + public var messageId: String? + + public init(messageId: String? = nil) { + self.messageId = messageId + } +} diff --git a/src/gateway/node-command-policy.ts b/src/gateway/node-command-policy.ts index 6361fc3947..791ff63ce0 100644 --- a/src/gateway/node-command-policy.ts +++ b/src/gateway/node-command-policy.ts @@ -34,6 +34,8 @@ const MOTION_COMMANDS = ["motion.activity", "motion.pedometer"]; const SYSTEM_NOTIFY_COMMANDS = ["system.notify"]; +const CHAT_COMMANDS = ["chat.push"]; + const TALK_COMMANDS = ["talk.ptt.start", "talk.ptt.stop", "talk.ptt.cancel", "talk.ptt.once"]; const SYSTEM_COMMANDS = [ @@ -52,6 +54,7 @@ const PLATFORM_DEFAULTS: Record = { ...SCREEN_COMMANDS, ...LOCATION_COMMANDS, ...SYSTEM_NOTIFY_COMMANDS, + ...CHAT_COMMANDS, ...DEVICE_COMMANDS, ...PHOTOS_COMMANDS, ...CONTACTS_COMMANDS,