iOS: wire node commands and incremental TTS

This commit is contained in:
Mariano Belinky
2026-02-01 12:21:10 +01:00
committed by Mariano Belinky
parent b7aac92ac4
commit 532b9653be
12 changed files with 1002 additions and 67 deletions

View File

@@ -0,0 +1,25 @@
import Foundation
import OpenClawKit
@MainActor
final class NodeCapabilityRouter {
enum RouterError: Error {
case unknownCommand
case handlerUnavailable
}
typealias Handler = (BridgeInvokeRequest) async throws -> BridgeInvokeResponse
private let handlers: [String: Handler]
init(handlers: [String: Handler]) {
self.handlers = handlers
}
func handle(_ request: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
guard let handler = handlers[request.command] else {
throw RouterError.unknownCommand
}
return try await handler(request)
}
}

View File

@@ -214,8 +214,8 @@ final class GatewayConnectionController {
guard let appModel else { return }
let connectOptions = self.makeConnectOptions()
Task { [weak self] in
guard let self else { return }
Task { [weak appModel] in
guard let appModel else { return }
await MainActor.run {
appModel.gatewayStatusText = "Connecting…"
}
@@ -353,6 +353,7 @@ final class GatewayConnectionController {
OpenClawCanvasA2UICommand.reset.rawValue,
OpenClawScreenCommand.record.rawValue,
OpenClawSystemCommand.notify.rawValue,
OpenClawChatCommand.push.rawValue,
OpenClawTalkCommand.pttStart.rawValue,
OpenClawTalkCommand.pttStop.rawValue,
OpenClawTalkCommand.pttCancel.rawValue,

View File

@@ -0,0 +1,85 @@
import Foundation
import OpenClawKit
@MainActor
final class GatewayHealthMonitor {
struct Config: Sendable {
var intervalSeconds: Double
var timeoutSeconds: Double
var maxFailures: Int
}
private let config: Config
private let sleep: @Sendable (UInt64) async -> Void
private var task: Task<Void, Never>?
init(
config: Config = Config(intervalSeconds: 15, timeoutSeconds: 5, maxFailures: 3),
sleep: @escaping @Sendable (UInt64) async -> Void = { nanoseconds in
try? await Task.sleep(nanoseconds: nanoseconds)
}
) {
self.config = config
self.sleep = sleep
}
func start(
check: @escaping @Sendable () async throws -> Bool,
onFailure: @escaping @Sendable (_ failureCount: Int) async -> Void)
{
self.stop()
let config = self.config
let sleep = self.sleep
self.task = Task { @MainActor in
var failures = 0
while !Task.isCancelled {
let ok = await Self.runCheck(check: check, timeoutSeconds: config.timeoutSeconds)
if ok {
failures = 0
} else {
failures += 1
if failures >= max(1, config.maxFailures) {
await onFailure(failures)
failures = 0
}
}
if Task.isCancelled { break }
let interval = max(0.0, config.intervalSeconds)
let nanos = UInt64(interval * 1_000_000_000)
if nanos > 0 {
await sleep(nanos)
} else {
await Task.yield()
}
}
}
}
func stop() {
self.task?.cancel()
self.task = nil
}
private static func runCheck(
check: @escaping @Sendable () async throws -> Bool,
timeoutSeconds: Double) async -> Bool
{
let timeout = max(0.0, timeoutSeconds)
if timeout == 0 {
return (try? await check()) ?? false
}
do {
let timeoutError = NSError(
domain: "GatewayHealthMonitor",
code: 1,
userInfo: [NSLocalizedDescriptionKey: "health check timed out"])
return try await AsyncTimeout.withTimeout(
seconds: timeout,
onTimeout: { timeoutError },
operation: check)
} catch {
return false
}
}
}

View File

@@ -1,3 +1,4 @@
import OpenClawChatUI
import OpenClawKit
import Observation
import SwiftUI
@@ -61,6 +62,8 @@ final class NodeAppModel {
private var gatewayTask: Task<Void, Never>?
private var voiceWakeSyncTask: Task<Void, Never>?
@ObservationIgnored private var cameraHUDDismissTask: Task<Void, Never>?
@ObservationIgnored private var capabilityRouter: NodeCapabilityRouter
private let gatewayHealthMonitor = GatewayHealthMonitor()
private let notificationCenter: NotificationCentering
let voiceWake = VoiceWakeManager()
let talkMode: TalkModeManager
@@ -108,6 +111,8 @@ final class NodeAppModel {
self.remindersService = remindersService
self.motionService = motionService
self.talkMode = talkMode
self.capabilityRouter = NodeCapabilityRouter(handlers: [:])
self.capabilityRouter = self.buildCapabilityRouter()
self.voiceWake.configure { [weak self] cmd in
guard let self else { return }
@@ -281,6 +286,7 @@ final class NodeAppModel {
connectOptions: GatewayConnectOptions)
{
self.gatewayTask?.cancel()
self.gatewayHealthMonitor.stop()
self.gatewayServerName = nil
self.gatewayRemoteAddress = nil
let id = gatewayStableID.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -325,6 +331,7 @@ final class NodeAppModel {
}
await self.refreshBrandingFromGateway()
await self.startVoiceWakeSync()
await MainActor.run { self.startGatewayHealthMonitor() }
await self.showA2UIOnConnectIfNeeded()
},
onDisconnected: { [weak self] reason in
@@ -337,6 +344,7 @@ final class NodeAppModel {
self.showLocalCanvasOnDisconnect()
self.gatewayStatusText = "Disconnected: \(reason)"
}
await MainActor.run { self.stopGatewayHealthMonitor() }
},
onInvoke: { [weak self] req in
guard let self else {
@@ -391,6 +399,7 @@ final class NodeAppModel {
self.gatewayTask = nil
self.voiceWakeSyncTask?.cancel()
self.voiceWakeSyncTask = nil
self.gatewayHealthMonitor.stop()
Task { await self.gateway.disconnect() }
self.gatewayStatusText = "Offline"
self.gatewayServerName = nil
@@ -492,6 +501,27 @@ final class NodeAppModel {
}
}
private func startGatewayHealthMonitor() {
self.gatewayHealthMonitor.start(
check: { [weak self] in
guard let self else { return false }
do {
let data = try await self.gateway.request(method: "health", paramsJSON: nil, timeoutSeconds: 6)
return (try? JSONDecoder().decode(OpenClawGatewayHealthOK.self, from: data))?.ok ?? true
} catch {
return false
}
},
onFailure: { [weak self] _ in
guard let self else { return }
await self.gateway.disconnect()
})
}
private func stopGatewayHealthMonitor() {
self.gatewayHealthMonitor.stop()
}
private func refreshWakeWordsFromGateway() async {
do {
let data = try await self.gateway.request(method: "voicewake.get", paramsJSON: "{}", timeoutSeconds: 8)
@@ -597,54 +627,19 @@ final class NodeAppModel {
}
do {
switch command {
case OpenClawLocationCommand.get.rawValue:
return try await self.handleLocationInvoke(req)
case OpenClawCanvasCommand.present.rawValue,
OpenClawCanvasCommand.hide.rawValue,
OpenClawCanvasCommand.navigate.rawValue,
OpenClawCanvasCommand.evalJS.rawValue,
OpenClawCanvasCommand.snapshot.rawValue:
return try await self.handleCanvasInvoke(req)
case OpenClawCanvasA2UICommand.reset.rawValue,
OpenClawCanvasA2UICommand.push.rawValue,
OpenClawCanvasA2UICommand.pushJSONL.rawValue:
return try await self.handleCanvasA2UIInvoke(req)
case OpenClawCameraCommand.list.rawValue,
OpenClawCameraCommand.snap.rawValue,
OpenClawCameraCommand.clip.rawValue:
return try await self.handleCameraInvoke(req)
case OpenClawScreenCommand.record.rawValue:
return try await self.handleScreenRecordInvoke(req)
case OpenClawSystemCommand.notify.rawValue:
return try await self.handleSystemNotify(req)
case OpenClawDeviceCommand.status.rawValue,
OpenClawDeviceCommand.info.rawValue:
return try await self.handleDeviceInvoke(req)
case OpenClawPhotosCommand.latest.rawValue:
return try await self.handlePhotosInvoke(req)
case OpenClawContactsCommand.search.rawValue,
OpenClawContactsCommand.add.rawValue:
return try await self.handleContactsInvoke(req)
case OpenClawCalendarCommand.events.rawValue,
OpenClawCalendarCommand.add.rawValue:
return try await self.handleCalendarInvoke(req)
case OpenClawRemindersCommand.list.rawValue,
OpenClawRemindersCommand.add.rawValue:
return try await self.handleRemindersInvoke(req)
case OpenClawMotionCommand.activity.rawValue,
OpenClawMotionCommand.pedometer.rawValue:
return try await self.handleMotionInvoke(req)
case OpenClawTalkCommand.pttStart.rawValue,
OpenClawTalkCommand.pttStop.rawValue,
OpenClawTalkCommand.pttCancel.rawValue,
OpenClawTalkCommand.pttOnce.rawValue:
return try await self.handleTalkInvoke(req)
default:
return try await self.capabilityRouter.handle(req)
} catch let error as NodeCapabilityRouter.RouterError {
switch error {
case .unknownCommand:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
case .handlerUnavailable:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: OpenClawNodeError(code: .unavailable, message: "node handler unavailable"))
}
} catch {
if command.hasPrefix("camera.") {
@@ -983,6 +978,22 @@ final class NodeAppModel {
let content = UNMutableNotificationContent()
content.title = title
content.body = body
if #available(iOS 15.0, *) {
switch params.priority ?? .active {
case .passive:
content.interruptionLevel = .passive
case .timeSensitive:
content.interruptionLevel = .timeSensitive
case .active:
content.interruptionLevel = .active
}
}
let soundValue = params.sound?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
if let soundValue, ["none", "silent", "off", "false", "0"].contains(soundValue) {
content.sound = nil
} else {
content.sound = .default
}
let request = UNNotificationRequest(
identifier: UUID().uuidString,
content: content,
@@ -998,6 +1009,51 @@ final class NodeAppModel {
return BridgeInvokeResponse(id: req.id, ok: true)
}
private func handleChatPushInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
let params = try Self.decodeParams(OpenClawChatPushParams.self, from: req.paramsJSON)
let text = params.text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !text.isEmpty else {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: empty chat.push text"))
}
let finalStatus = await self.requestNotificationAuthorizationIfNeeded()
let messageId = UUID().uuidString
if finalStatus == .authorized || finalStatus == .provisional || finalStatus == .ephemeral {
let addResult = await self.runNotificationCall(timeoutSeconds: 2.0) { [notificationCenter] in
let content = UNMutableNotificationContent()
content.title = "OpenClaw"
content.body = text
content.sound = .default
content.userInfo = ["messageId": messageId]
let request = UNNotificationRequest(
identifier: messageId,
content: content,
trigger: nil)
try await notificationCenter.add(request)
}
if case let .failure(error) = addResult {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: OpenClawNodeError(code: .unavailable, message: "NOTIFICATION_FAILED: \(error.message)"))
}
}
if params.speak ?? true {
let toSpeak = text
Task { @MainActor in
try? await TalkSystemSpeechSynthesizer.shared.speak(text: toSpeak)
}
}
let payload = OpenClawChatPushPayload(messageId: messageId)
let json = try Self.encodePayload(payload)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
}
private func requestNotificationAuthorizationIfNeeded() async -> NotificationAuthorizationStatus {
let status = await self.notificationAuthorizationStatus()
guard status == .notDetermined else { return status }
@@ -1203,6 +1259,123 @@ final class NodeAppModel {
}
private extension NodeAppModel {
// Central registry for node invoke routing to keep commands in one place.
func buildCapabilityRouter() -> NodeCapabilityRouter {
var handlers: [String: NodeCapabilityRouter.Handler] = [:]
func register(_ commands: [String], handler: @escaping NodeCapabilityRouter.Handler) {
for command in commands {
handlers[command] = handler
}
}
register([OpenClawLocationCommand.get.rawValue]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handleLocationInvoke(req)
}
register([
OpenClawCanvasCommand.present.rawValue,
OpenClawCanvasCommand.hide.rawValue,
OpenClawCanvasCommand.navigate.rawValue,
OpenClawCanvasCommand.evalJS.rawValue,
OpenClawCanvasCommand.snapshot.rawValue,
]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handleCanvasInvoke(req)
}
register([
OpenClawCanvasA2UICommand.reset.rawValue,
OpenClawCanvasA2UICommand.push.rawValue,
OpenClawCanvasA2UICommand.pushJSONL.rawValue,
]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handleCanvasA2UIInvoke(req)
}
register([
OpenClawCameraCommand.list.rawValue,
OpenClawCameraCommand.snap.rawValue,
OpenClawCameraCommand.clip.rawValue,
]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handleCameraInvoke(req)
}
register([OpenClawScreenCommand.record.rawValue]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handleScreenRecordInvoke(req)
}
register([OpenClawSystemCommand.notify.rawValue]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handleSystemNotify(req)
}
register([OpenClawChatCommand.push.rawValue]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handleChatPushInvoke(req)
}
register([
OpenClawDeviceCommand.status.rawValue,
OpenClawDeviceCommand.info.rawValue,
]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handleDeviceInvoke(req)
}
register([OpenClawPhotosCommand.latest.rawValue]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handlePhotosInvoke(req)
}
register([
OpenClawContactsCommand.search.rawValue,
OpenClawContactsCommand.add.rawValue,
]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handleContactsInvoke(req)
}
register([
OpenClawCalendarCommand.events.rawValue,
OpenClawCalendarCommand.add.rawValue,
]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handleCalendarInvoke(req)
}
register([
OpenClawRemindersCommand.list.rawValue,
OpenClawRemindersCommand.add.rawValue,
]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handleRemindersInvoke(req)
}
register([
OpenClawMotionCommand.activity.rawValue,
OpenClawMotionCommand.pedometer.rawValue,
]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handleMotionInvoke(req)
}
register([
OpenClawTalkCommand.pttStart.rawValue,
OpenClawTalkCommand.pttStop.rawValue,
OpenClawTalkCommand.pttCancel.rawValue,
OpenClawTalkCommand.pttOnce.rawValue,
]) { [weak self] req in
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
return try await self.handleTalkInvoke(req)
}
return NodeCapabilityRouter(handlers: handlers)
}
func locationMode() -> OpenClawLocationMode {
let raw = UserDefaults.standard.string(forKey: "location.enabledMode") ?? "off"
return OpenClawLocationMode(rawValue: raw) ?? .off

View File

@@ -1,4 +1,5 @@
import AVFAudio
import OpenClawChatUI
import OpenClawKit
import OpenClawProtocol
import Foundation
@@ -65,6 +66,14 @@ final class TalkModeManager: NSObject {
private let silenceWindow: TimeInterval = 0.7
private var chatSubscribedSessionKeys = Set<String>()
private var incrementalSpeechQueue: [String] = []
private var incrementalSpeechTask: Task<Void, Never>?
private var incrementalSpeechActive = false
private var incrementalSpeechUsed = false
private var incrementalSpeechLanguage: String?
private var incrementalSpeechBuffer = IncrementalSpeechBuffer()
private var incrementalSpeechContext: IncrementalSpeechContext?
private var incrementalSpeechDirective: TalkDirective?
private let logger = Logger(subsystem: "bot.molt", category: "TalkMode")
@@ -456,6 +465,14 @@ final class TalkModeManager: NSObject {
}
if isFinal {
self.lastTranscript = trimmed
guard !trimmed.isEmpty else { return }
if self.captureMode == .pushToTalk, self.pttAutoStopEnabled, self.isPushToTalkActive {
_ = await self.endPushToTalk()
return
}
if self.captureMode == .continuous, !self.isSpeaking {
await self.processTranscript(trimmed, restartAfter: true)
}
}
}
@@ -539,6 +556,15 @@ final class TalkModeManager: NSObject {
"chat.send start sessionKey=\(sessionKey, privacy: .public) chars=\(prompt.count, privacy: .public)")
let runId = try await self.sendChat(prompt, gateway: gateway)
self.logger.info("chat.send ok runId=\(runId, privacy: .public)")
let shouldIncremental = self.shouldUseIncrementalTTS()
var streamingTask: Task<Void, Never>?
if shouldIncremental {
self.resetIncrementalSpeech()
streamingTask = Task { @MainActor [weak self] in
guard let self else { return }
await self.streamAssistant(runId: runId, gateway: gateway)
}
}
let completion = await self.waitForChatCompletion(runId: runId, gateway: gateway, timeoutSeconds: 120)
if completion == .timeout {
self.logger.warning(
@@ -546,27 +572,44 @@ final class TalkModeManager: NSObject {
} else if completion == .aborted {
self.statusText = "Aborted"
self.logger.warning("chat completion aborted runId=\(runId, privacy: .public)")
streamingTask?.cancel()
await self.finishIncrementalSpeech()
await self.start()
return
} else if completion == .error {
self.statusText = "Chat error"
self.logger.warning("chat completion error runId=\(runId, privacy: .public)")
streamingTask?.cancel()
await self.finishIncrementalSpeech()
await self.start()
return
}
guard let assistantText = try await self.waitForAssistantText(
var assistantText = try await self.waitForAssistantText(
gateway: gateway,
since: startedAt,
timeoutSeconds: completion == .final ? 12 : 25)
else {
if assistantText == nil, shouldIncremental {
let fallback = self.incrementalSpeechBuffer.latestText
if !fallback.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
assistantText = fallback
}
}
guard let assistantText else {
self.statusText = "No reply"
self.logger.warning("assistant text timeout runId=\(runId, privacy: .public)")
streamingTask?.cancel()
await self.finishIncrementalSpeech()
await self.start()
return
}
self.logger.info("assistant text ok chars=\(assistantText.count, privacy: .public)")
await self.playAssistant(text: assistantText)
streamingTask?.cancel()
if shouldIncremental {
await self.handleIncrementalAssistantFinal(text: assistantText)
} else {
await self.playAssistant(text: assistantText)
}
} catch {
self.statusText = "Talk failed: \(error.localizedDescription)"
self.logger.error("finalize failed: \(error.localizedDescription, privacy: .public)")
@@ -720,24 +763,7 @@ final class TalkModeManager: NSObject {
let directive = parsed.directive
let cleaned = parsed.stripped.trimmingCharacters(in: .whitespacesAndNewlines)
guard !cleaned.isEmpty else { return }
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
}
if let voice = resolvedVoice {
if directive?.once != true {
self.currentVoiceId = voice
self.voiceOverrideActive = true
}
}
if let model = directive?.modelId {
if directive?.once != true {
self.currentModelId = model
self.modelOverrideActive = true
}
}
self.applyDirective(directive)
self.statusText = "Generating voice…"
self.isSpeaking = true
@@ -746,6 +772,11 @@ final class TalkModeManager: NSObject {
do {
let started = Date()
let language = ElevenLabsTTSClient.validatedLanguage(directive?.language)
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
}
let resolvedKey =
(self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ??
@@ -875,6 +906,7 @@ final class TalkModeManager: NSObject {
? self.mp3Player.stop()
: self.pcmPlayer.stop()
TalkSystemSpeechSynthesizer.shared.stop()
self.cancelIncrementalSpeech()
self.isSpeaking = false
}
@@ -887,6 +919,268 @@ final class TalkModeManager: NSObject {
return true
}
private func shouldUseIncrementalTTS() -> Bool {
true
}
private func applyDirective(_ directive: TalkDirective?) {
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
}
if let voice = resolvedVoice {
if directive?.once != true {
self.currentVoiceId = voice
self.voiceOverrideActive = true
}
}
if let model = directive?.modelId {
if directive?.once != true {
self.currentModelId = model
self.modelOverrideActive = true
}
}
}
private func resetIncrementalSpeech() {
self.incrementalSpeechQueue.removeAll()
self.incrementalSpeechTask?.cancel()
self.incrementalSpeechTask = nil
self.incrementalSpeechActive = true
self.incrementalSpeechUsed = false
self.incrementalSpeechLanguage = nil
self.incrementalSpeechBuffer = IncrementalSpeechBuffer()
self.incrementalSpeechContext = nil
self.incrementalSpeechDirective = nil
}
private func cancelIncrementalSpeech() {
self.incrementalSpeechQueue.removeAll()
self.incrementalSpeechTask?.cancel()
self.incrementalSpeechTask = nil
self.incrementalSpeechActive = false
self.incrementalSpeechContext = nil
self.incrementalSpeechDirective = nil
}
private func enqueueIncrementalSpeech(_ text: String) {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
self.incrementalSpeechQueue.append(trimmed)
self.incrementalSpeechUsed = true
if self.incrementalSpeechTask == nil {
self.startIncrementalSpeechTask()
}
}
private func startIncrementalSpeechTask() {
if self.interruptOnSpeech {
do {
try self.startRecognition()
} catch {
self.logger.warning(
"startRecognition during incremental speak failed: \(error.localizedDescription, privacy: .public)")
}
}
self.incrementalSpeechTask = Task { @MainActor [weak self] in
guard let self else { return }
while !Task.isCancelled {
guard !self.incrementalSpeechQueue.isEmpty else { break }
let segment = self.incrementalSpeechQueue.removeFirst()
self.statusText = "Speaking…"
self.isSpeaking = true
self.lastSpokenText = segment
await self.speakIncrementalSegment(segment)
}
self.isSpeaking = false
self.stopRecognition()
self.incrementalSpeechTask = nil
}
}
private func finishIncrementalSpeech() async {
guard self.incrementalSpeechActive else { return }
let leftover = self.incrementalSpeechBuffer.flush()
if let leftover {
self.enqueueIncrementalSpeech(leftover)
}
if let task = self.incrementalSpeechTask {
_ = await task.result
}
self.incrementalSpeechActive = false
}
private func handleIncrementalAssistantFinal(text: String) async {
let parsed = TalkDirectiveParser.parse(text)
self.applyDirective(parsed.directive)
if let lang = parsed.directive?.language {
self.incrementalSpeechLanguage = ElevenLabsTTSClient.validatedLanguage(lang)
}
await self.updateIncrementalContextIfNeeded()
let segments = self.incrementalSpeechBuffer.ingest(text: text, isFinal: true)
for segment in segments {
self.enqueueIncrementalSpeech(segment)
}
await self.finishIncrementalSpeech()
if !self.incrementalSpeechUsed {
await self.playAssistant(text: text)
}
}
private func streamAssistant(runId: String, gateway: GatewayNodeSession) async {
let stream = await gateway.subscribeServerEvents(bufferingNewest: 200)
for await evt in stream {
if Task.isCancelled { return }
guard evt.event == "agent", let payload = evt.payload else { continue }
guard let agentEvent = try? GatewayPayloadDecoding.decode(payload, as: OpenClawAgentEventPayload.self) else {
continue
}
guard agentEvent.runId == runId, agentEvent.stream == "assistant" else { continue }
guard let text = agentEvent.data["text"]?.value as? String else { continue }
let segments = self.incrementalSpeechBuffer.ingest(text: text, isFinal: false)
if let lang = self.incrementalSpeechBuffer.directive?.language {
self.incrementalSpeechLanguage = ElevenLabsTTSClient.validatedLanguage(lang)
}
await self.updateIncrementalContextIfNeeded()
for segment in segments {
self.enqueueIncrementalSpeech(segment)
}
}
}
private func updateIncrementalContextIfNeeded() async {
let directive = self.incrementalSpeechBuffer.directive
if let existing = self.incrementalSpeechContext, directive == self.incrementalSpeechDirective {
if existing.language != self.incrementalSpeechLanguage {
self.incrementalSpeechContext = IncrementalSpeechContext(
apiKey: existing.apiKey,
voiceId: existing.voiceId,
modelId: existing.modelId,
outputFormat: existing.outputFormat,
language: self.incrementalSpeechLanguage,
directive: existing.directive,
canUseElevenLabs: existing.canUseElevenLabs)
}
return
}
let context = await self.buildIncrementalSpeechContext(directive: directive)
self.incrementalSpeechContext = context
self.incrementalSpeechDirective = directive
}
private func buildIncrementalSpeechContext(directive: TalkDirective?) async -> IncrementalSpeechContext {
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
}
let preferredVoice = resolvedVoice ?? self.currentVoiceId ?? self.defaultVoiceId
let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId
let desiredOutputFormat = (directive?.outputFormat ?? self.defaultOutputFormat)?
.trimmingCharacters(in: .whitespacesAndNewlines)
let requestedOutputFormat = (desiredOutputFormat?.isEmpty == false) ? desiredOutputFormat : nil
let outputFormat = ElevenLabsTTSClient.validatedOutputFormat(requestedOutputFormat ?? "pcm_44100")
if outputFormat == nil, let requestedOutputFormat {
self.logger.warning(
"talk output_format unsupported for local playback: \(requestedOutputFormat, privacy: .public)")
}
let resolvedKey =
(self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ??
ProcessInfo.processInfo.environment["ELEVENLABS_API_KEY"]
let apiKey = resolvedKey?.trimmingCharacters(in: .whitespacesAndNewlines)
let voiceId: String? = if let apiKey, !apiKey.isEmpty {
await self.resolveVoiceId(preferred: preferredVoice, apiKey: apiKey)
} else {
nil
}
let canUseElevenLabs = (voiceId?.isEmpty == false) && (apiKey?.isEmpty == false)
return IncrementalSpeechContext(
apiKey: apiKey,
voiceId: voiceId,
modelId: modelId,
outputFormat: outputFormat,
language: self.incrementalSpeechLanguage,
directive: directive,
canUseElevenLabs: canUseElevenLabs)
}
private func speakIncrementalSegment(_ text: String) async {
await self.updateIncrementalContextIfNeeded()
guard let context = self.incrementalSpeechContext else {
try? await TalkSystemSpeechSynthesizer.shared.speak(
text: text,
language: self.incrementalSpeechLanguage)
return
}
if context.canUseElevenLabs, let apiKey = context.apiKey, let voiceId = context.voiceId {
let request = ElevenLabsTTSRequest(
text: text,
modelId: context.modelId,
outputFormat: context.outputFormat,
speed: TalkTTSValidation.resolveSpeed(
speed: context.directive?.speed,
rateWPM: context.directive?.rateWPM),
stability: TalkTTSValidation.validatedStability(
context.directive?.stability,
modelId: context.modelId),
similarity: TalkTTSValidation.validatedUnit(context.directive?.similarity),
style: TalkTTSValidation.validatedUnit(context.directive?.style),
speakerBoost: context.directive?.speakerBoost,
seed: TalkTTSValidation.validatedSeed(context.directive?.seed),
normalize: ElevenLabsTTSClient.validatedNormalize(context.directive?.normalize),
language: context.language,
latencyTier: TalkTTSValidation.validatedLatencyTier(context.directive?.latencyTier))
let client = ElevenLabsTTSClient(apiKey: apiKey)
let stream = client.streamSynthesize(voiceId: voiceId, request: request)
let sampleRate = TalkTTSValidation.pcmSampleRate(from: context.outputFormat)
let result: StreamingPlaybackResult
if let sampleRate {
self.lastPlaybackWasPCM = true
var playback = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate)
if !playback.finished, playback.interruptedAt == nil {
self.logger.warning("pcm playback failed; retrying mp3")
self.lastPlaybackWasPCM = false
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100")
let mp3Stream = client.streamSynthesize(
voiceId: voiceId,
request: ElevenLabsTTSRequest(
text: text,
modelId: context.modelId,
outputFormat: mp3Format,
speed: TalkTTSValidation.resolveSpeed(
speed: context.directive?.speed,
rateWPM: context.directive?.rateWPM),
stability: TalkTTSValidation.validatedStability(
context.directive?.stability,
modelId: context.modelId),
similarity: TalkTTSValidation.validatedUnit(context.directive?.similarity),
style: TalkTTSValidation.validatedUnit(context.directive?.style),
speakerBoost: context.directive?.speakerBoost,
seed: TalkTTSValidation.validatedSeed(context.directive?.seed),
normalize: ElevenLabsTTSClient.validatedNormalize(context.directive?.normalize),
language: context.language,
latencyTier: TalkTTSValidation.validatedLatencyTier(context.directive?.latencyTier)))
playback = await self.mp3Player.play(stream: mp3Stream)
}
result = playback
} else {
self.lastPlaybackWasPCM = false
result = await self.mp3Player.play(stream: stream)
}
if !result.finished, let interruptedAt = result.interruptedAt {
self.lastInterruptedAtSeconds = interruptedAt
}
} else {
try? await TalkSystemSpeechSynthesizer.shared.speak(
text: text,
language: self.incrementalSpeechLanguage)
}
}
private func resolveVoiceAlias(_ value: String?) -> String? {
let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
@@ -1010,6 +1304,121 @@ final class TalkModeManager: NSObject {
}
}
private struct IncrementalSpeechBuffer {
private(set) var latestText: String = ""
private(set) var directive: TalkDirective?
private var spokenOffset: Int = 0
private var inCodeBlock = false
private var directiveParsed = false
mutating func ingest(text: String, isFinal: Bool) -> [String] {
let normalized = text.replacingOccurrences(of: "\r\n", with: "\n")
guard let usable = self.stripDirectiveIfReady(from: normalized) else { return [] }
self.updateText(usable)
return self.extractSegments(isFinal: isFinal)
}
mutating func flush() -> String? {
guard !self.latestText.isEmpty else { return nil }
let segments = self.extractSegments(isFinal: true)
return segments.first
}
private mutating func stripDirectiveIfReady(from text: String) -> String? {
guard !self.directiveParsed else { return text }
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
if trimmed.hasPrefix("{") {
guard let newlineRange = text.range(of: "\n") else { return nil }
let firstLine = text[..<newlineRange.lowerBound]
let head = firstLine.trimmingCharacters(in: .whitespacesAndNewlines)
guard head.hasSuffix("}") else { return nil }
let parsed = TalkDirectiveParser.parse(text)
if let directive = parsed.directive {
self.directive = directive
}
self.directiveParsed = true
return parsed.stripped
}
self.directiveParsed = true
return text
}
private mutating func updateText(_ newText: String) {
if newText.hasPrefix(self.latestText) {
self.latestText = newText
} else if self.latestText.hasPrefix(newText) {
// Keep the longer cached text.
} else {
self.latestText += newText
}
if self.spokenOffset > self.latestText.count {
self.spokenOffset = self.latestText.count
}
}
private mutating func extractSegments(isFinal: Bool) -> [String] {
let chars = Array(self.latestText)
guard self.spokenOffset < chars.count else { return [] }
var idx = self.spokenOffset
var lastBoundary: Int?
var inCodeBlock = self.inCodeBlock
var buffer = ""
var bufferAtBoundary = ""
var inCodeBlockAtBoundary = inCodeBlock
while idx < chars.count {
if idx + 2 < chars.count,
chars[idx] == "`",
chars[idx + 1] == "`",
chars[idx + 2] == "`"
{
inCodeBlock.toggle()
idx += 3
continue
}
if !inCodeBlock {
buffer.append(chars[idx])
if Self.isBoundary(chars[idx]) {
lastBoundary = idx + 1
bufferAtBoundary = buffer
inCodeBlockAtBoundary = inCodeBlock
}
}
idx += 1
}
if let boundary = lastBoundary {
self.spokenOffset = boundary
self.inCodeBlock = inCodeBlockAtBoundary
let trimmed = bufferAtBoundary.trimmingCharacters(in: .whitespacesAndNewlines)
return trimmed.isEmpty ? [] : [trimmed]
}
guard isFinal else { return [] }
self.spokenOffset = chars.count
self.inCodeBlock = inCodeBlock
let trimmed = buffer.trimmingCharacters(in: .whitespacesAndNewlines)
return trimmed.isEmpty ? [] : [trimmed]
}
private static func isBoundary(_ ch: Character) -> Bool {
ch == "." || ch == "!" || ch == "?" || ch == "\n"
}
}
private struct IncrementalSpeechContext {
let apiKey: String?
let voiceId: String?
let modelId: String?
let outputFormat: String?
let language: String?
let directive: TalkDirective?
let canUseElevenLabs: Bool
}
#if DEBUG
extension TalkModeManager {
func _test_seedTranscript(_ transcript: String) {
@@ -1017,6 +1426,10 @@ extension TalkModeManager {
self.lastHeard = Date()
}
func _test_handleTranscript(_ transcript: String, isFinal: Bool) async {
await self.handleTranscript(transcript: transcript, isFinal: isFinal)
}
func _test_backdateLastHeard(seconds: TimeInterval) {
self.lastHeard = Date().addingTimeInterval(-seconds)
}
@@ -1024,5 +1437,13 @@ extension TalkModeManager {
func _test_runSilenceCheck() async {
await self.checkSilence()
}
func _test_incrementalReset() {
self.incrementalSpeechBuffer = IncrementalSpeechBuffer()
}
func _test_incrementalIngest(_ text: String, isFinal: Bool) -> [String] {
self.incrementalSpeechBuffer.ingest(text: text, isFinal: isFinal)
}
}
#endif

View File

@@ -2,7 +2,9 @@ Sources/Gateway/GatewayConnectionController.swift
Sources/Gateway/GatewayDiscoveryDebugLogView.swift
Sources/Gateway/GatewayDiscoveryModel.swift
Sources/Gateway/GatewaySettingsStore.swift
Sources/Gateway/GatewayHealthMonitor.swift
Sources/Gateway/KeychainStore.swift
Sources/Capabilities/NodeCapabilityRouter.swift
Sources/Camera/CameraController.swift
Sources/Chat/ChatSheet.swift
Sources/Chat/IOSGatewayChatTransport.swift
@@ -58,6 +60,7 @@ Sources/Voice/VoiceWakePreferences.swift
../shared/OpenClawKit/Sources/OpenClawKit/CanvasCommandParams.swift
../shared/OpenClawKit/Sources/OpenClawKit/CanvasCommands.swift
../shared/OpenClawKit/Sources/OpenClawKit/Capabilities.swift
../shared/OpenClawKit/Sources/OpenClawKit/ChatCommands.swift
../shared/OpenClawKit/Sources/OpenClawKit/ContactsCommands.swift
../shared/OpenClawKit/Sources/OpenClawKit/DeviceCommands.swift
../shared/OpenClawKit/Sources/OpenClawKit/OpenClawKitResources.swift

View File

@@ -92,6 +92,7 @@ private func withUserDefaults<T>(_ updates: [String: Any?], _ body: () throws ->
let commands = Set(controller._test_currentCommands())
#expect(commands.contains(OpenClawSystemCommand.notify.rawValue))
#expect(commands.contains(OpenClawChatCommand.push.rawValue))
#expect(!commands.contains(OpenClawSystemCommand.run.rawValue))
#expect(!commands.contains(OpenClawSystemCommand.which.rawValue))
#expect(!commands.contains(OpenClawSystemCommand.execApprovalsGet.rawValue))

View File

@@ -0,0 +1,60 @@
import Foundation
import Testing
@testable import OpenClaw
private actor Counter {
private var value = 0
func increment() {
value += 1
}
func get() -> Int {
value
}
func set(_ newValue: Int) {
value = newValue
}
}
@Suite struct GatewayHealthMonitorTests {
@Test @MainActor func triggersFailureAfterThreshold() async {
let failureCount = Counter()
let monitor = GatewayHealthMonitor(
config: .init(intervalSeconds: 0.001, timeoutSeconds: 0.0, maxFailures: 2))
monitor.start(
check: { false },
onFailure: { _ in
await failureCount.increment()
await monitor.stop()
})
try? await Task.sleep(nanoseconds: 60_000_000)
#expect(await failureCount.get() == 1)
}
@Test @MainActor func resetsFailuresAfterSuccess() async {
let failureCount = Counter()
let calls = Counter()
let monitor = GatewayHealthMonitor(
config: .init(intervalSeconds: 0.001, timeoutSeconds: 0.0, maxFailures: 2))
monitor.start(
check: {
await calls.increment()
let callCount = await calls.get()
if callCount >= 6 {
await monitor.stop()
}
return callCount % 2 == 0
},
onFailure: { _ in
await failureCount.increment()
})
try? await Task.sleep(nanoseconds: 60_000_000)
#expect(await failureCount.get() == 0)
}
}

View File

@@ -448,6 +448,91 @@ private func decodePayload<T: Decodable>(_ json: String?, as type: T.Type) throw
#expect(request.content.body == "World")
}
@Test @MainActor func handleInvokeChatPushCreatesNotification() async throws {
let notifier = TestNotificationCenter(status: .authorized)
let deviceStatus = TestDeviceStatusService(
statusPayload: OpenClawDeviceStatusPayload(
battery: OpenClawBatteryStatusPayload(level: 0.5, state: .charging, lowPowerModeEnabled: false),
thermal: OpenClawThermalStatusPayload(state: .nominal),
storage: OpenClawStorageStatusPayload(totalBytes: 100, freeBytes: 50, usedBytes: 50),
network: OpenClawNetworkStatusPayload(
status: .satisfied,
isExpensive: false,
isConstrained: false,
interfaces: [.wifi]),
uptimeSeconds: 10),
infoPayload: OpenClawDeviceInfoPayload(
deviceName: "Test",
modelIdentifier: "Test1,1",
systemName: "iOS",
systemVersion: "1.0",
appVersion: "dev",
appBuild: "0",
locale: "en-US"))
let emptyContact = OpenClawContactPayload(
identifier: "c0",
displayName: "",
givenName: "",
familyName: "",
organizationName: "",
phoneNumbers: [],
emails: [])
let emptyEvent = OpenClawCalendarEventPayload(
identifier: "e0",
title: "Test",
startISO: "2024-01-01T00:00:00Z",
endISO: "2024-01-01T00:30:00Z",
isAllDay: false,
location: nil,
calendarTitle: nil)
let emptyReminder = OpenClawReminderPayload(
identifier: "r0",
title: "Test",
dueISO: nil,
completed: false,
listName: nil)
let appModel = makeTestAppModel(
notificationCenter: notifier,
deviceStatusService: deviceStatus,
photosService: TestPhotosService(payload: OpenClawPhotosLatestPayload(photos: [])),
contactsService: TestContactsService(
searchPayload: OpenClawContactsSearchPayload(contacts: []),
addPayload: OpenClawContactsAddPayload(contact: emptyContact)),
calendarService: TestCalendarService(
eventsPayload: OpenClawCalendarEventsPayload(events: []),
addPayload: OpenClawCalendarAddPayload(event: emptyEvent)),
remindersService: TestRemindersService(
listPayload: OpenClawRemindersListPayload(reminders: []),
addPayload: OpenClawRemindersAddPayload(reminder: emptyReminder)),
motionService: TestMotionService(
activityPayload: OpenClawMotionActivityPayload(activities: []),
pedometerPayload: OpenClawPedometerPayload(
startISO: "2024-01-01T00:00:00Z",
endISO: "2024-01-01T01:00:00Z",
steps: nil,
distanceMeters: nil,
floorsAscended: nil,
floorsDescended: nil)))
let params = OpenClawChatPushParams(text: "Ping", speak: false)
let data = try JSONEncoder().encode(params)
let json = String(decoding: data, as: UTF8.self)
let req = BridgeInvokeRequest(
id: "chat-push",
command: OpenClawChatCommand.push.rawValue,
paramsJSON: json)
let res = await appModel._test_handleInvoke(req)
#expect(res.ok == true)
#expect(notifier.addedRequests.count == 1)
let request = try #require(notifier.addedRequests.first)
#expect(request.content.title == "OpenClaw")
#expect(request.content.body == "Ping")
let payloadJSON = try #require(res.payloadJSON)
let decoded = try JSONDecoder().decode(OpenClawChatPushPayload.self, from: Data(payloadJSON.utf8))
#expect((decoded.messageId ?? "").isEmpty == false)
#expect(request.identifier == decoded.messageId)
}
@Test @MainActor func handleInvokeDeviceAndDataCommandsReturnPayloads() async throws {
let deviceStatusPayload = OpenClawDeviceStatusPayload(
battery: OpenClawBatteryStatusPayload(level: 0.25, state: .unplugged, lowPowerModeEnabled: false),
@@ -723,6 +808,28 @@ private func decodePayload<T: Decodable>(_ json: String?, as type: T.Type) throw
#expect(oncePayload.status == "offline")
}
@Test @MainActor func handleInvokePushToTalkOnceStopsOnFinalTranscript() async throws {
let talkMode = TalkModeManager(allowSimulatorCapture: true)
talkMode.updateGatewayConnected(false)
let appModel = makeTalkTestAppModel(talkMode: talkMode)
let onceReq = BridgeInvokeRequest(id: "ptt-once-final", command: OpenClawTalkCommand.pttOnce.rawValue)
let onceTask = Task { await appModel._test_handleInvoke(onceReq) }
for _ in 0..<5 where !talkMode.isPushToTalkActive {
await Task.yield()
}
#expect(talkMode.isPushToTalkActive == true)
await talkMode._test_handleTranscript("Hello final", isFinal: true)
let onceRes = await onceTask.value
#expect(onceRes.ok == true)
let oncePayload = try decodePayload(onceRes.payloadJSON, as: OpenClawTalkPTTStopPayload.self)
#expect(oncePayload.transcript == "Hello final")
#expect(oncePayload.status == "offline")
}
@Test @MainActor func handleDeepLinkSetsErrorWhenNotConnected() async {
let appModel = NodeAppModel()
let url = URL(string: "openclaw://agent?message=hello")!

View File

@@ -0,0 +1,33 @@
import Foundation
import Testing
@testable import OpenClaw
@Suite struct TalkModeIncrementalTests {
@Test @MainActor func incrementalSpeechSplitsOnBoundary() {
let talkMode = TalkModeManager(allowSimulatorCapture: true)
talkMode._test_incrementalReset()
let segments = talkMode._test_incrementalIngest("Hello world. Next", isFinal: false)
#expect(segments.count == 1)
#expect(segments.first == "Hello world.")
}
@Test @MainActor func incrementalSpeechSkipsDirectiveLine() {
let talkMode = TalkModeManager(allowSimulatorCapture: true)
talkMode._test_incrementalReset()
let segments = talkMode._test_incrementalIngest("{\"voice\":\"abc\"}\nHello.", isFinal: false)
#expect(segments.count == 1)
#expect(segments.first == "Hello.")
}
@Test @MainActor func incrementalSpeechIgnoresCodeBlocks() {
let talkMode = TalkModeManager(allowSimulatorCapture: true)
talkMode._test_incrementalReset()
let text = "Here is code:\n```js\nx=1\n```\nDone."
let segments = talkMode._test_incrementalIngest(text, isFinal: true)
#expect(segments.count == 1)
let value = segments.first ?? ""
#expect(value.contains("x=1") == false)
#expect(value.contains("Here is code") == true)
#expect(value.contains("Done.") == true)
}
}

View File

@@ -0,0 +1,23 @@
import Foundation
public enum OpenClawChatCommand: String, Codable, Sendable {
case push = "chat.push"
}
public struct OpenClawChatPushParams: Codable, Sendable, Equatable {
public var text: String
public var speak: Bool?
public init(text: String, speak: Bool? = nil) {
self.text = text
self.speak = speak
}
}
public struct OpenClawChatPushPayload: Codable, Sendable, Equatable {
public var messageId: String?
public init(messageId: String? = nil) {
self.messageId = messageId
}
}

View File

@@ -34,6 +34,8 @@ const MOTION_COMMANDS = ["motion.activity", "motion.pedometer"];
const SYSTEM_NOTIFY_COMMANDS = ["system.notify"];
const CHAT_COMMANDS = ["chat.push"];
const TALK_COMMANDS = ["talk.ptt.start", "talk.ptt.stop", "talk.ptt.cancel", "talk.ptt.once"];
const SYSTEM_COMMANDS = [
@@ -52,6 +54,7 @@ const PLATFORM_DEFAULTS: Record<string, string[]> = {
...SCREEN_COMMANDS,
...LOCATION_COMMANDS,
...SYSTEM_NOTIFY_COMMANDS,
...CHAT_COMMANDS,
...DEVICE_COMMANDS,
...PHOTOS_COMMANDS,
...CONTACTS_COMMANDS,