mirror of
https://github.com/openclaw/openclaw.git
synced 2026-02-19 18:39:20 -05:00
iOS: wire node commands and incremental TTS
This commit is contained in:
committed by
Mariano Belinky
parent
b7aac92ac4
commit
532b9653be
25
apps/ios/Sources/Capabilities/NodeCapabilityRouter.swift
Normal file
25
apps/ios/Sources/Capabilities/NodeCapabilityRouter.swift
Normal file
@@ -0,0 +1,25 @@
|
||||
import Foundation
|
||||
import OpenClawKit
|
||||
|
||||
@MainActor
|
||||
final class NodeCapabilityRouter {
|
||||
enum RouterError: Error {
|
||||
case unknownCommand
|
||||
case handlerUnavailable
|
||||
}
|
||||
|
||||
typealias Handler = (BridgeInvokeRequest) async throws -> BridgeInvokeResponse
|
||||
|
||||
private let handlers: [String: Handler]
|
||||
|
||||
init(handlers: [String: Handler]) {
|
||||
self.handlers = handlers
|
||||
}
|
||||
|
||||
func handle(_ request: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
guard let handler = handlers[request.command] else {
|
||||
throw RouterError.unknownCommand
|
||||
}
|
||||
return try await handler(request)
|
||||
}
|
||||
}
|
||||
@@ -214,8 +214,8 @@ final class GatewayConnectionController {
|
||||
guard let appModel else { return }
|
||||
let connectOptions = self.makeConnectOptions()
|
||||
|
||||
Task { [weak self] in
|
||||
guard let self else { return }
|
||||
Task { [weak appModel] in
|
||||
guard let appModel else { return }
|
||||
await MainActor.run {
|
||||
appModel.gatewayStatusText = "Connecting…"
|
||||
}
|
||||
@@ -353,6 +353,7 @@ final class GatewayConnectionController {
|
||||
OpenClawCanvasA2UICommand.reset.rawValue,
|
||||
OpenClawScreenCommand.record.rawValue,
|
||||
OpenClawSystemCommand.notify.rawValue,
|
||||
OpenClawChatCommand.push.rawValue,
|
||||
OpenClawTalkCommand.pttStart.rawValue,
|
||||
OpenClawTalkCommand.pttStop.rawValue,
|
||||
OpenClawTalkCommand.pttCancel.rawValue,
|
||||
|
||||
85
apps/ios/Sources/Gateway/GatewayHealthMonitor.swift
Normal file
85
apps/ios/Sources/Gateway/GatewayHealthMonitor.swift
Normal file
@@ -0,0 +1,85 @@
|
||||
import Foundation
|
||||
import OpenClawKit
|
||||
|
||||
@MainActor
|
||||
final class GatewayHealthMonitor {
|
||||
struct Config: Sendable {
|
||||
var intervalSeconds: Double
|
||||
var timeoutSeconds: Double
|
||||
var maxFailures: Int
|
||||
}
|
||||
|
||||
private let config: Config
|
||||
private let sleep: @Sendable (UInt64) async -> Void
|
||||
private var task: Task<Void, Never>?
|
||||
|
||||
init(
|
||||
config: Config = Config(intervalSeconds: 15, timeoutSeconds: 5, maxFailures: 3),
|
||||
sleep: @escaping @Sendable (UInt64) async -> Void = { nanoseconds in
|
||||
try? await Task.sleep(nanoseconds: nanoseconds)
|
||||
}
|
||||
) {
|
||||
self.config = config
|
||||
self.sleep = sleep
|
||||
}
|
||||
|
||||
func start(
|
||||
check: @escaping @Sendable () async throws -> Bool,
|
||||
onFailure: @escaping @Sendable (_ failureCount: Int) async -> Void)
|
||||
{
|
||||
self.stop()
|
||||
let config = self.config
|
||||
let sleep = self.sleep
|
||||
self.task = Task { @MainActor in
|
||||
var failures = 0
|
||||
while !Task.isCancelled {
|
||||
let ok = await Self.runCheck(check: check, timeoutSeconds: config.timeoutSeconds)
|
||||
if ok {
|
||||
failures = 0
|
||||
} else {
|
||||
failures += 1
|
||||
if failures >= max(1, config.maxFailures) {
|
||||
await onFailure(failures)
|
||||
failures = 0
|
||||
}
|
||||
}
|
||||
|
||||
if Task.isCancelled { break }
|
||||
let interval = max(0.0, config.intervalSeconds)
|
||||
let nanos = UInt64(interval * 1_000_000_000)
|
||||
if nanos > 0 {
|
||||
await sleep(nanos)
|
||||
} else {
|
||||
await Task.yield()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
self.task?.cancel()
|
||||
self.task = nil
|
||||
}
|
||||
|
||||
private static func runCheck(
|
||||
check: @escaping @Sendable () async throws -> Bool,
|
||||
timeoutSeconds: Double) async -> Bool
|
||||
{
|
||||
let timeout = max(0.0, timeoutSeconds)
|
||||
if timeout == 0 {
|
||||
return (try? await check()) ?? false
|
||||
}
|
||||
do {
|
||||
let timeoutError = NSError(
|
||||
domain: "GatewayHealthMonitor",
|
||||
code: 1,
|
||||
userInfo: [NSLocalizedDescriptionKey: "health check timed out"])
|
||||
return try await AsyncTimeout.withTimeout(
|
||||
seconds: timeout,
|
||||
onTimeout: { timeoutError },
|
||||
operation: check)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import OpenClawChatUI
|
||||
import OpenClawKit
|
||||
import Observation
|
||||
import SwiftUI
|
||||
@@ -61,6 +62,8 @@ final class NodeAppModel {
|
||||
private var gatewayTask: Task<Void, Never>?
|
||||
private var voiceWakeSyncTask: Task<Void, Never>?
|
||||
@ObservationIgnored private var cameraHUDDismissTask: Task<Void, Never>?
|
||||
@ObservationIgnored private var capabilityRouter: NodeCapabilityRouter
|
||||
private let gatewayHealthMonitor = GatewayHealthMonitor()
|
||||
private let notificationCenter: NotificationCentering
|
||||
let voiceWake = VoiceWakeManager()
|
||||
let talkMode: TalkModeManager
|
||||
@@ -108,6 +111,8 @@ final class NodeAppModel {
|
||||
self.remindersService = remindersService
|
||||
self.motionService = motionService
|
||||
self.talkMode = talkMode
|
||||
self.capabilityRouter = NodeCapabilityRouter(handlers: [:])
|
||||
self.capabilityRouter = self.buildCapabilityRouter()
|
||||
|
||||
self.voiceWake.configure { [weak self] cmd in
|
||||
guard let self else { return }
|
||||
@@ -281,6 +286,7 @@ final class NodeAppModel {
|
||||
connectOptions: GatewayConnectOptions)
|
||||
{
|
||||
self.gatewayTask?.cancel()
|
||||
self.gatewayHealthMonitor.stop()
|
||||
self.gatewayServerName = nil
|
||||
self.gatewayRemoteAddress = nil
|
||||
let id = gatewayStableID.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
@@ -325,6 +331,7 @@ final class NodeAppModel {
|
||||
}
|
||||
await self.refreshBrandingFromGateway()
|
||||
await self.startVoiceWakeSync()
|
||||
await MainActor.run { self.startGatewayHealthMonitor() }
|
||||
await self.showA2UIOnConnectIfNeeded()
|
||||
},
|
||||
onDisconnected: { [weak self] reason in
|
||||
@@ -337,6 +344,7 @@ final class NodeAppModel {
|
||||
self.showLocalCanvasOnDisconnect()
|
||||
self.gatewayStatusText = "Disconnected: \(reason)"
|
||||
}
|
||||
await MainActor.run { self.stopGatewayHealthMonitor() }
|
||||
},
|
||||
onInvoke: { [weak self] req in
|
||||
guard let self else {
|
||||
@@ -391,6 +399,7 @@ final class NodeAppModel {
|
||||
self.gatewayTask = nil
|
||||
self.voiceWakeSyncTask?.cancel()
|
||||
self.voiceWakeSyncTask = nil
|
||||
self.gatewayHealthMonitor.stop()
|
||||
Task { await self.gateway.disconnect() }
|
||||
self.gatewayStatusText = "Offline"
|
||||
self.gatewayServerName = nil
|
||||
@@ -492,6 +501,27 @@ final class NodeAppModel {
|
||||
}
|
||||
}
|
||||
|
||||
private func startGatewayHealthMonitor() {
|
||||
self.gatewayHealthMonitor.start(
|
||||
check: { [weak self] in
|
||||
guard let self else { return false }
|
||||
do {
|
||||
let data = try await self.gateway.request(method: "health", paramsJSON: nil, timeoutSeconds: 6)
|
||||
return (try? JSONDecoder().decode(OpenClawGatewayHealthOK.self, from: data))?.ok ?? true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
},
|
||||
onFailure: { [weak self] _ in
|
||||
guard let self else { return }
|
||||
await self.gateway.disconnect()
|
||||
})
|
||||
}
|
||||
|
||||
private func stopGatewayHealthMonitor() {
|
||||
self.gatewayHealthMonitor.stop()
|
||||
}
|
||||
|
||||
private func refreshWakeWordsFromGateway() async {
|
||||
do {
|
||||
let data = try await self.gateway.request(method: "voicewake.get", paramsJSON: "{}", timeoutSeconds: 8)
|
||||
@@ -597,54 +627,19 @@ final class NodeAppModel {
|
||||
}
|
||||
|
||||
do {
|
||||
switch command {
|
||||
case OpenClawLocationCommand.get.rawValue:
|
||||
return try await self.handleLocationInvoke(req)
|
||||
case OpenClawCanvasCommand.present.rawValue,
|
||||
OpenClawCanvasCommand.hide.rawValue,
|
||||
OpenClawCanvasCommand.navigate.rawValue,
|
||||
OpenClawCanvasCommand.evalJS.rawValue,
|
||||
OpenClawCanvasCommand.snapshot.rawValue:
|
||||
return try await self.handleCanvasInvoke(req)
|
||||
case OpenClawCanvasA2UICommand.reset.rawValue,
|
||||
OpenClawCanvasA2UICommand.push.rawValue,
|
||||
OpenClawCanvasA2UICommand.pushJSONL.rawValue:
|
||||
return try await self.handleCanvasA2UIInvoke(req)
|
||||
case OpenClawCameraCommand.list.rawValue,
|
||||
OpenClawCameraCommand.snap.rawValue,
|
||||
OpenClawCameraCommand.clip.rawValue:
|
||||
return try await self.handleCameraInvoke(req)
|
||||
case OpenClawScreenCommand.record.rawValue:
|
||||
return try await self.handleScreenRecordInvoke(req)
|
||||
case OpenClawSystemCommand.notify.rawValue:
|
||||
return try await self.handleSystemNotify(req)
|
||||
case OpenClawDeviceCommand.status.rawValue,
|
||||
OpenClawDeviceCommand.info.rawValue:
|
||||
return try await self.handleDeviceInvoke(req)
|
||||
case OpenClawPhotosCommand.latest.rawValue:
|
||||
return try await self.handlePhotosInvoke(req)
|
||||
case OpenClawContactsCommand.search.rawValue,
|
||||
OpenClawContactsCommand.add.rawValue:
|
||||
return try await self.handleContactsInvoke(req)
|
||||
case OpenClawCalendarCommand.events.rawValue,
|
||||
OpenClawCalendarCommand.add.rawValue:
|
||||
return try await self.handleCalendarInvoke(req)
|
||||
case OpenClawRemindersCommand.list.rawValue,
|
||||
OpenClawRemindersCommand.add.rawValue:
|
||||
return try await self.handleRemindersInvoke(req)
|
||||
case OpenClawMotionCommand.activity.rawValue,
|
||||
OpenClawMotionCommand.pedometer.rawValue:
|
||||
return try await self.handleMotionInvoke(req)
|
||||
case OpenClawTalkCommand.pttStart.rawValue,
|
||||
OpenClawTalkCommand.pttStop.rawValue,
|
||||
OpenClawTalkCommand.pttCancel.rawValue,
|
||||
OpenClawTalkCommand.pttOnce.rawValue:
|
||||
return try await self.handleTalkInvoke(req)
|
||||
default:
|
||||
return try await self.capabilityRouter.handle(req)
|
||||
} catch let error as NodeCapabilityRouter.RouterError {
|
||||
switch error {
|
||||
case .unknownCommand:
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
|
||||
case .handlerUnavailable:
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: OpenClawNodeError(code: .unavailable, message: "node handler unavailable"))
|
||||
}
|
||||
} catch {
|
||||
if command.hasPrefix("camera.") {
|
||||
@@ -983,6 +978,22 @@ final class NodeAppModel {
|
||||
let content = UNMutableNotificationContent()
|
||||
content.title = title
|
||||
content.body = body
|
||||
if #available(iOS 15.0, *) {
|
||||
switch params.priority ?? .active {
|
||||
case .passive:
|
||||
content.interruptionLevel = .passive
|
||||
case .timeSensitive:
|
||||
content.interruptionLevel = .timeSensitive
|
||||
case .active:
|
||||
content.interruptionLevel = .active
|
||||
}
|
||||
}
|
||||
let soundValue = params.sound?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
||||
if let soundValue, ["none", "silent", "off", "false", "0"].contains(soundValue) {
|
||||
content.sound = nil
|
||||
} else {
|
||||
content.sound = .default
|
||||
}
|
||||
let request = UNNotificationRequest(
|
||||
identifier: UUID().uuidString,
|
||||
content: content,
|
||||
@@ -998,6 +1009,51 @@ final class NodeAppModel {
|
||||
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||
}
|
||||
|
||||
private func handleChatPushInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
let params = try Self.decodeParams(OpenClawChatPushParams.self, from: req.paramsJSON)
|
||||
let text = params.text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !text.isEmpty else {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: empty chat.push text"))
|
||||
}
|
||||
|
||||
let finalStatus = await self.requestNotificationAuthorizationIfNeeded()
|
||||
let messageId = UUID().uuidString
|
||||
if finalStatus == .authorized || finalStatus == .provisional || finalStatus == .ephemeral {
|
||||
let addResult = await self.runNotificationCall(timeoutSeconds: 2.0) { [notificationCenter] in
|
||||
let content = UNMutableNotificationContent()
|
||||
content.title = "OpenClaw"
|
||||
content.body = text
|
||||
content.sound = .default
|
||||
content.userInfo = ["messageId": messageId]
|
||||
let request = UNNotificationRequest(
|
||||
identifier: messageId,
|
||||
content: content,
|
||||
trigger: nil)
|
||||
try await notificationCenter.add(request)
|
||||
}
|
||||
if case let .failure(error) = addResult {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: OpenClawNodeError(code: .unavailable, message: "NOTIFICATION_FAILED: \(error.message)"))
|
||||
}
|
||||
}
|
||||
|
||||
if params.speak ?? true {
|
||||
let toSpeak = text
|
||||
Task { @MainActor in
|
||||
try? await TalkSystemSpeechSynthesizer.shared.speak(text: toSpeak)
|
||||
}
|
||||
}
|
||||
|
||||
let payload = OpenClawChatPushPayload(messageId: messageId)
|
||||
let json = try Self.encodePayload(payload)
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
|
||||
}
|
||||
|
||||
private func requestNotificationAuthorizationIfNeeded() async -> NotificationAuthorizationStatus {
|
||||
let status = await self.notificationAuthorizationStatus()
|
||||
guard status == .notDetermined else { return status }
|
||||
@@ -1203,6 +1259,123 @@ final class NodeAppModel {
|
||||
}
|
||||
|
||||
private extension NodeAppModel {
|
||||
// Central registry for node invoke routing to keep commands in one place.
|
||||
func buildCapabilityRouter() -> NodeCapabilityRouter {
|
||||
var handlers: [String: NodeCapabilityRouter.Handler] = [:]
|
||||
|
||||
func register(_ commands: [String], handler: @escaping NodeCapabilityRouter.Handler) {
|
||||
for command in commands {
|
||||
handlers[command] = handler
|
||||
}
|
||||
}
|
||||
|
||||
register([OpenClawLocationCommand.get.rawValue]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handleLocationInvoke(req)
|
||||
}
|
||||
|
||||
register([
|
||||
OpenClawCanvasCommand.present.rawValue,
|
||||
OpenClawCanvasCommand.hide.rawValue,
|
||||
OpenClawCanvasCommand.navigate.rawValue,
|
||||
OpenClawCanvasCommand.evalJS.rawValue,
|
||||
OpenClawCanvasCommand.snapshot.rawValue,
|
||||
]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handleCanvasInvoke(req)
|
||||
}
|
||||
|
||||
register([
|
||||
OpenClawCanvasA2UICommand.reset.rawValue,
|
||||
OpenClawCanvasA2UICommand.push.rawValue,
|
||||
OpenClawCanvasA2UICommand.pushJSONL.rawValue,
|
||||
]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handleCanvasA2UIInvoke(req)
|
||||
}
|
||||
|
||||
register([
|
||||
OpenClawCameraCommand.list.rawValue,
|
||||
OpenClawCameraCommand.snap.rawValue,
|
||||
OpenClawCameraCommand.clip.rawValue,
|
||||
]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handleCameraInvoke(req)
|
||||
}
|
||||
|
||||
register([OpenClawScreenCommand.record.rawValue]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handleScreenRecordInvoke(req)
|
||||
}
|
||||
|
||||
register([OpenClawSystemCommand.notify.rawValue]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handleSystemNotify(req)
|
||||
}
|
||||
|
||||
register([OpenClawChatCommand.push.rawValue]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handleChatPushInvoke(req)
|
||||
}
|
||||
|
||||
register([
|
||||
OpenClawDeviceCommand.status.rawValue,
|
||||
OpenClawDeviceCommand.info.rawValue,
|
||||
]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handleDeviceInvoke(req)
|
||||
}
|
||||
|
||||
register([OpenClawPhotosCommand.latest.rawValue]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handlePhotosInvoke(req)
|
||||
}
|
||||
|
||||
register([
|
||||
OpenClawContactsCommand.search.rawValue,
|
||||
OpenClawContactsCommand.add.rawValue,
|
||||
]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handleContactsInvoke(req)
|
||||
}
|
||||
|
||||
register([
|
||||
OpenClawCalendarCommand.events.rawValue,
|
||||
OpenClawCalendarCommand.add.rawValue,
|
||||
]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handleCalendarInvoke(req)
|
||||
}
|
||||
|
||||
register([
|
||||
OpenClawRemindersCommand.list.rawValue,
|
||||
OpenClawRemindersCommand.add.rawValue,
|
||||
]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handleRemindersInvoke(req)
|
||||
}
|
||||
|
||||
register([
|
||||
OpenClawMotionCommand.activity.rawValue,
|
||||
OpenClawMotionCommand.pedometer.rawValue,
|
||||
]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handleMotionInvoke(req)
|
||||
}
|
||||
|
||||
register([
|
||||
OpenClawTalkCommand.pttStart.rawValue,
|
||||
OpenClawTalkCommand.pttStop.rawValue,
|
||||
OpenClawTalkCommand.pttCancel.rawValue,
|
||||
OpenClawTalkCommand.pttOnce.rawValue,
|
||||
]) { [weak self] req in
|
||||
guard let self else { throw NodeCapabilityRouter.RouterError.handlerUnavailable }
|
||||
return try await self.handleTalkInvoke(req)
|
||||
}
|
||||
|
||||
return NodeCapabilityRouter(handlers: handlers)
|
||||
}
|
||||
|
||||
func locationMode() -> OpenClawLocationMode {
|
||||
let raw = UserDefaults.standard.string(forKey: "location.enabledMode") ?? "off"
|
||||
return OpenClawLocationMode(rawValue: raw) ?? .off
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import AVFAudio
|
||||
import OpenClawChatUI
|
||||
import OpenClawKit
|
||||
import OpenClawProtocol
|
||||
import Foundation
|
||||
@@ -65,6 +66,14 @@ final class TalkModeManager: NSObject {
|
||||
private let silenceWindow: TimeInterval = 0.7
|
||||
|
||||
private var chatSubscribedSessionKeys = Set<String>()
|
||||
private var incrementalSpeechQueue: [String] = []
|
||||
private var incrementalSpeechTask: Task<Void, Never>?
|
||||
private var incrementalSpeechActive = false
|
||||
private var incrementalSpeechUsed = false
|
||||
private var incrementalSpeechLanguage: String?
|
||||
private var incrementalSpeechBuffer = IncrementalSpeechBuffer()
|
||||
private var incrementalSpeechContext: IncrementalSpeechContext?
|
||||
private var incrementalSpeechDirective: TalkDirective?
|
||||
|
||||
private let logger = Logger(subsystem: "bot.molt", category: "TalkMode")
|
||||
|
||||
@@ -456,6 +465,14 @@ final class TalkModeManager: NSObject {
|
||||
}
|
||||
if isFinal {
|
||||
self.lastTranscript = trimmed
|
||||
guard !trimmed.isEmpty else { return }
|
||||
if self.captureMode == .pushToTalk, self.pttAutoStopEnabled, self.isPushToTalkActive {
|
||||
_ = await self.endPushToTalk()
|
||||
return
|
||||
}
|
||||
if self.captureMode == .continuous, !self.isSpeaking {
|
||||
await self.processTranscript(trimmed, restartAfter: true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -539,6 +556,15 @@ final class TalkModeManager: NSObject {
|
||||
"chat.send start sessionKey=\(sessionKey, privacy: .public) chars=\(prompt.count, privacy: .public)")
|
||||
let runId = try await self.sendChat(prompt, gateway: gateway)
|
||||
self.logger.info("chat.send ok runId=\(runId, privacy: .public)")
|
||||
let shouldIncremental = self.shouldUseIncrementalTTS()
|
||||
var streamingTask: Task<Void, Never>?
|
||||
if shouldIncremental {
|
||||
self.resetIncrementalSpeech()
|
||||
streamingTask = Task { @MainActor [weak self] in
|
||||
guard let self else { return }
|
||||
await self.streamAssistant(runId: runId, gateway: gateway)
|
||||
}
|
||||
}
|
||||
let completion = await self.waitForChatCompletion(runId: runId, gateway: gateway, timeoutSeconds: 120)
|
||||
if completion == .timeout {
|
||||
self.logger.warning(
|
||||
@@ -546,27 +572,44 @@ final class TalkModeManager: NSObject {
|
||||
} else if completion == .aborted {
|
||||
self.statusText = "Aborted"
|
||||
self.logger.warning("chat completion aborted runId=\(runId, privacy: .public)")
|
||||
streamingTask?.cancel()
|
||||
await self.finishIncrementalSpeech()
|
||||
await self.start()
|
||||
return
|
||||
} else if completion == .error {
|
||||
self.statusText = "Chat error"
|
||||
self.logger.warning("chat completion error runId=\(runId, privacy: .public)")
|
||||
streamingTask?.cancel()
|
||||
await self.finishIncrementalSpeech()
|
||||
await self.start()
|
||||
return
|
||||
}
|
||||
|
||||
guard let assistantText = try await self.waitForAssistantText(
|
||||
var assistantText = try await self.waitForAssistantText(
|
||||
gateway: gateway,
|
||||
since: startedAt,
|
||||
timeoutSeconds: completion == .final ? 12 : 25)
|
||||
else {
|
||||
if assistantText == nil, shouldIncremental {
|
||||
let fallback = self.incrementalSpeechBuffer.latestText
|
||||
if !fallback.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
|
||||
assistantText = fallback
|
||||
}
|
||||
}
|
||||
guard let assistantText else {
|
||||
self.statusText = "No reply"
|
||||
self.logger.warning("assistant text timeout runId=\(runId, privacy: .public)")
|
||||
streamingTask?.cancel()
|
||||
await self.finishIncrementalSpeech()
|
||||
await self.start()
|
||||
return
|
||||
}
|
||||
self.logger.info("assistant text ok chars=\(assistantText.count, privacy: .public)")
|
||||
await self.playAssistant(text: assistantText)
|
||||
streamingTask?.cancel()
|
||||
if shouldIncremental {
|
||||
await self.handleIncrementalAssistantFinal(text: assistantText)
|
||||
} else {
|
||||
await self.playAssistant(text: assistantText)
|
||||
}
|
||||
} catch {
|
||||
self.statusText = "Talk failed: \(error.localizedDescription)"
|
||||
self.logger.error("finalize failed: \(error.localizedDescription, privacy: .public)")
|
||||
@@ -720,24 +763,7 @@ final class TalkModeManager: NSObject {
|
||||
let directive = parsed.directive
|
||||
let cleaned = parsed.stripped.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !cleaned.isEmpty else { return }
|
||||
|
||||
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
|
||||
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
|
||||
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
|
||||
}
|
||||
if let voice = resolvedVoice {
|
||||
if directive?.once != true {
|
||||
self.currentVoiceId = voice
|
||||
self.voiceOverrideActive = true
|
||||
}
|
||||
}
|
||||
if let model = directive?.modelId {
|
||||
if directive?.once != true {
|
||||
self.currentModelId = model
|
||||
self.modelOverrideActive = true
|
||||
}
|
||||
}
|
||||
self.applyDirective(directive)
|
||||
|
||||
self.statusText = "Generating voice…"
|
||||
self.isSpeaking = true
|
||||
@@ -746,6 +772,11 @@ final class TalkModeManager: NSObject {
|
||||
do {
|
||||
let started = Date()
|
||||
let language = ElevenLabsTTSClient.validatedLanguage(directive?.language)
|
||||
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
|
||||
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
|
||||
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
|
||||
}
|
||||
|
||||
let resolvedKey =
|
||||
(self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ??
|
||||
@@ -875,6 +906,7 @@ final class TalkModeManager: NSObject {
|
||||
? self.mp3Player.stop()
|
||||
: self.pcmPlayer.stop()
|
||||
TalkSystemSpeechSynthesizer.shared.stop()
|
||||
self.cancelIncrementalSpeech()
|
||||
self.isSpeaking = false
|
||||
}
|
||||
|
||||
@@ -887,6 +919,268 @@ final class TalkModeManager: NSObject {
|
||||
return true
|
||||
}
|
||||
|
||||
private func shouldUseIncrementalTTS() -> Bool {
|
||||
true
|
||||
}
|
||||
|
||||
private func applyDirective(_ directive: TalkDirective?) {
|
||||
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
|
||||
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
|
||||
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
|
||||
}
|
||||
if let voice = resolvedVoice {
|
||||
if directive?.once != true {
|
||||
self.currentVoiceId = voice
|
||||
self.voiceOverrideActive = true
|
||||
}
|
||||
}
|
||||
if let model = directive?.modelId {
|
||||
if directive?.once != true {
|
||||
self.currentModelId = model
|
||||
self.modelOverrideActive = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func resetIncrementalSpeech() {
|
||||
self.incrementalSpeechQueue.removeAll()
|
||||
self.incrementalSpeechTask?.cancel()
|
||||
self.incrementalSpeechTask = nil
|
||||
self.incrementalSpeechActive = true
|
||||
self.incrementalSpeechUsed = false
|
||||
self.incrementalSpeechLanguage = nil
|
||||
self.incrementalSpeechBuffer = IncrementalSpeechBuffer()
|
||||
self.incrementalSpeechContext = nil
|
||||
self.incrementalSpeechDirective = nil
|
||||
}
|
||||
|
||||
private func cancelIncrementalSpeech() {
|
||||
self.incrementalSpeechQueue.removeAll()
|
||||
self.incrementalSpeechTask?.cancel()
|
||||
self.incrementalSpeechTask = nil
|
||||
self.incrementalSpeechActive = false
|
||||
self.incrementalSpeechContext = nil
|
||||
self.incrementalSpeechDirective = nil
|
||||
}
|
||||
|
||||
private func enqueueIncrementalSpeech(_ text: String) {
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return }
|
||||
self.incrementalSpeechQueue.append(trimmed)
|
||||
self.incrementalSpeechUsed = true
|
||||
if self.incrementalSpeechTask == nil {
|
||||
self.startIncrementalSpeechTask()
|
||||
}
|
||||
}
|
||||
|
||||
private func startIncrementalSpeechTask() {
|
||||
if self.interruptOnSpeech {
|
||||
do {
|
||||
try self.startRecognition()
|
||||
} catch {
|
||||
self.logger.warning(
|
||||
"startRecognition during incremental speak failed: \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
}
|
||||
|
||||
self.incrementalSpeechTask = Task { @MainActor [weak self] in
|
||||
guard let self else { return }
|
||||
while !Task.isCancelled {
|
||||
guard !self.incrementalSpeechQueue.isEmpty else { break }
|
||||
let segment = self.incrementalSpeechQueue.removeFirst()
|
||||
self.statusText = "Speaking…"
|
||||
self.isSpeaking = true
|
||||
self.lastSpokenText = segment
|
||||
await self.speakIncrementalSegment(segment)
|
||||
}
|
||||
self.isSpeaking = false
|
||||
self.stopRecognition()
|
||||
self.incrementalSpeechTask = nil
|
||||
}
|
||||
}
|
||||
|
||||
private func finishIncrementalSpeech() async {
|
||||
guard self.incrementalSpeechActive else { return }
|
||||
let leftover = self.incrementalSpeechBuffer.flush()
|
||||
if let leftover {
|
||||
self.enqueueIncrementalSpeech(leftover)
|
||||
}
|
||||
if let task = self.incrementalSpeechTask {
|
||||
_ = await task.result
|
||||
}
|
||||
self.incrementalSpeechActive = false
|
||||
}
|
||||
|
||||
private func handleIncrementalAssistantFinal(text: String) async {
|
||||
let parsed = TalkDirectiveParser.parse(text)
|
||||
self.applyDirective(parsed.directive)
|
||||
if let lang = parsed.directive?.language {
|
||||
self.incrementalSpeechLanguage = ElevenLabsTTSClient.validatedLanguage(lang)
|
||||
}
|
||||
await self.updateIncrementalContextIfNeeded()
|
||||
let segments = self.incrementalSpeechBuffer.ingest(text: text, isFinal: true)
|
||||
for segment in segments {
|
||||
self.enqueueIncrementalSpeech(segment)
|
||||
}
|
||||
await self.finishIncrementalSpeech()
|
||||
if !self.incrementalSpeechUsed {
|
||||
await self.playAssistant(text: text)
|
||||
}
|
||||
}
|
||||
|
||||
private func streamAssistant(runId: String, gateway: GatewayNodeSession) async {
|
||||
let stream = await gateway.subscribeServerEvents(bufferingNewest: 200)
|
||||
for await evt in stream {
|
||||
if Task.isCancelled { return }
|
||||
guard evt.event == "agent", let payload = evt.payload else { continue }
|
||||
guard let agentEvent = try? GatewayPayloadDecoding.decode(payload, as: OpenClawAgentEventPayload.self) else {
|
||||
continue
|
||||
}
|
||||
guard agentEvent.runId == runId, agentEvent.stream == "assistant" else { continue }
|
||||
guard let text = agentEvent.data["text"]?.value as? String else { continue }
|
||||
let segments = self.incrementalSpeechBuffer.ingest(text: text, isFinal: false)
|
||||
if let lang = self.incrementalSpeechBuffer.directive?.language {
|
||||
self.incrementalSpeechLanguage = ElevenLabsTTSClient.validatedLanguage(lang)
|
||||
}
|
||||
await self.updateIncrementalContextIfNeeded()
|
||||
for segment in segments {
|
||||
self.enqueueIncrementalSpeech(segment)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func updateIncrementalContextIfNeeded() async {
|
||||
let directive = self.incrementalSpeechBuffer.directive
|
||||
if let existing = self.incrementalSpeechContext, directive == self.incrementalSpeechDirective {
|
||||
if existing.language != self.incrementalSpeechLanguage {
|
||||
self.incrementalSpeechContext = IncrementalSpeechContext(
|
||||
apiKey: existing.apiKey,
|
||||
voiceId: existing.voiceId,
|
||||
modelId: existing.modelId,
|
||||
outputFormat: existing.outputFormat,
|
||||
language: self.incrementalSpeechLanguage,
|
||||
directive: existing.directive,
|
||||
canUseElevenLabs: existing.canUseElevenLabs)
|
||||
}
|
||||
return
|
||||
}
|
||||
let context = await self.buildIncrementalSpeechContext(directive: directive)
|
||||
self.incrementalSpeechContext = context
|
||||
self.incrementalSpeechDirective = directive
|
||||
}
|
||||
|
||||
private func buildIncrementalSpeechContext(directive: TalkDirective?) async -> IncrementalSpeechContext {
|
||||
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
|
||||
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
|
||||
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
|
||||
}
|
||||
let preferredVoice = resolvedVoice ?? self.currentVoiceId ?? self.defaultVoiceId
|
||||
let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId
|
||||
let desiredOutputFormat = (directive?.outputFormat ?? self.defaultOutputFormat)?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let requestedOutputFormat = (desiredOutputFormat?.isEmpty == false) ? desiredOutputFormat : nil
|
||||
let outputFormat = ElevenLabsTTSClient.validatedOutputFormat(requestedOutputFormat ?? "pcm_44100")
|
||||
if outputFormat == nil, let requestedOutputFormat {
|
||||
self.logger.warning(
|
||||
"talk output_format unsupported for local playback: \(requestedOutputFormat, privacy: .public)")
|
||||
}
|
||||
|
||||
let resolvedKey =
|
||||
(self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ??
|
||||
ProcessInfo.processInfo.environment["ELEVENLABS_API_KEY"]
|
||||
let apiKey = resolvedKey?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let voiceId: String? = if let apiKey, !apiKey.isEmpty {
|
||||
await self.resolveVoiceId(preferred: preferredVoice, apiKey: apiKey)
|
||||
} else {
|
||||
nil
|
||||
}
|
||||
let canUseElevenLabs = (voiceId?.isEmpty == false) && (apiKey?.isEmpty == false)
|
||||
return IncrementalSpeechContext(
|
||||
apiKey: apiKey,
|
||||
voiceId: voiceId,
|
||||
modelId: modelId,
|
||||
outputFormat: outputFormat,
|
||||
language: self.incrementalSpeechLanguage,
|
||||
directive: directive,
|
||||
canUseElevenLabs: canUseElevenLabs)
|
||||
}
|
||||
|
||||
private func speakIncrementalSegment(_ text: String) async {
|
||||
await self.updateIncrementalContextIfNeeded()
|
||||
guard let context = self.incrementalSpeechContext else {
|
||||
try? await TalkSystemSpeechSynthesizer.shared.speak(
|
||||
text: text,
|
||||
language: self.incrementalSpeechLanguage)
|
||||
return
|
||||
}
|
||||
|
||||
if context.canUseElevenLabs, let apiKey = context.apiKey, let voiceId = context.voiceId {
|
||||
let request = ElevenLabsTTSRequest(
|
||||
text: text,
|
||||
modelId: context.modelId,
|
||||
outputFormat: context.outputFormat,
|
||||
speed: TalkTTSValidation.resolveSpeed(
|
||||
speed: context.directive?.speed,
|
||||
rateWPM: context.directive?.rateWPM),
|
||||
stability: TalkTTSValidation.validatedStability(
|
||||
context.directive?.stability,
|
||||
modelId: context.modelId),
|
||||
similarity: TalkTTSValidation.validatedUnit(context.directive?.similarity),
|
||||
style: TalkTTSValidation.validatedUnit(context.directive?.style),
|
||||
speakerBoost: context.directive?.speakerBoost,
|
||||
seed: TalkTTSValidation.validatedSeed(context.directive?.seed),
|
||||
normalize: ElevenLabsTTSClient.validatedNormalize(context.directive?.normalize),
|
||||
language: context.language,
|
||||
latencyTier: TalkTTSValidation.validatedLatencyTier(context.directive?.latencyTier))
|
||||
let client = ElevenLabsTTSClient(apiKey: apiKey)
|
||||
let stream = client.streamSynthesize(voiceId: voiceId, request: request)
|
||||
let sampleRate = TalkTTSValidation.pcmSampleRate(from: context.outputFormat)
|
||||
let result: StreamingPlaybackResult
|
||||
if let sampleRate {
|
||||
self.lastPlaybackWasPCM = true
|
||||
var playback = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate)
|
||||
if !playback.finished, playback.interruptedAt == nil {
|
||||
self.logger.warning("pcm playback failed; retrying mp3")
|
||||
self.lastPlaybackWasPCM = false
|
||||
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100")
|
||||
let mp3Stream = client.streamSynthesize(
|
||||
voiceId: voiceId,
|
||||
request: ElevenLabsTTSRequest(
|
||||
text: text,
|
||||
modelId: context.modelId,
|
||||
outputFormat: mp3Format,
|
||||
speed: TalkTTSValidation.resolveSpeed(
|
||||
speed: context.directive?.speed,
|
||||
rateWPM: context.directive?.rateWPM),
|
||||
stability: TalkTTSValidation.validatedStability(
|
||||
context.directive?.stability,
|
||||
modelId: context.modelId),
|
||||
similarity: TalkTTSValidation.validatedUnit(context.directive?.similarity),
|
||||
style: TalkTTSValidation.validatedUnit(context.directive?.style),
|
||||
speakerBoost: context.directive?.speakerBoost,
|
||||
seed: TalkTTSValidation.validatedSeed(context.directive?.seed),
|
||||
normalize: ElevenLabsTTSClient.validatedNormalize(context.directive?.normalize),
|
||||
language: context.language,
|
||||
latencyTier: TalkTTSValidation.validatedLatencyTier(context.directive?.latencyTier)))
|
||||
playback = await self.mp3Player.play(stream: mp3Stream)
|
||||
}
|
||||
result = playback
|
||||
} else {
|
||||
self.lastPlaybackWasPCM = false
|
||||
result = await self.mp3Player.play(stream: stream)
|
||||
}
|
||||
if !result.finished, let interruptedAt = result.interruptedAt {
|
||||
self.lastInterruptedAtSeconds = interruptedAt
|
||||
}
|
||||
} else {
|
||||
try? await TalkSystemSpeechSynthesizer.shared.speak(
|
||||
text: text,
|
||||
language: self.incrementalSpeechLanguage)
|
||||
}
|
||||
}
|
||||
|
||||
private func resolveVoiceAlias(_ value: String?) -> String? {
|
||||
let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return nil }
|
||||
@@ -1010,6 +1304,121 @@ final class TalkModeManager: NSObject {
|
||||
}
|
||||
}
|
||||
|
||||
private struct IncrementalSpeechBuffer {
|
||||
private(set) var latestText: String = ""
|
||||
private(set) var directive: TalkDirective?
|
||||
private var spokenOffset: Int = 0
|
||||
private var inCodeBlock = false
|
||||
private var directiveParsed = false
|
||||
|
||||
mutating func ingest(text: String, isFinal: Bool) -> [String] {
|
||||
let normalized = text.replacingOccurrences(of: "\r\n", with: "\n")
|
||||
guard let usable = self.stripDirectiveIfReady(from: normalized) else { return [] }
|
||||
self.updateText(usable)
|
||||
return self.extractSegments(isFinal: isFinal)
|
||||
}
|
||||
|
||||
mutating func flush() -> String? {
|
||||
guard !self.latestText.isEmpty else { return nil }
|
||||
let segments = self.extractSegments(isFinal: true)
|
||||
return segments.first
|
||||
}
|
||||
|
||||
private mutating func stripDirectiveIfReady(from text: String) -> String? {
|
||||
guard !self.directiveParsed else { return text }
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return nil }
|
||||
if trimmed.hasPrefix("{") {
|
||||
guard let newlineRange = text.range(of: "\n") else { return nil }
|
||||
let firstLine = text[..<newlineRange.lowerBound]
|
||||
let head = firstLine.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard head.hasSuffix("}") else { return nil }
|
||||
let parsed = TalkDirectiveParser.parse(text)
|
||||
if let directive = parsed.directive {
|
||||
self.directive = directive
|
||||
}
|
||||
self.directiveParsed = true
|
||||
return parsed.stripped
|
||||
}
|
||||
self.directiveParsed = true
|
||||
return text
|
||||
}
|
||||
|
||||
private mutating func updateText(_ newText: String) {
|
||||
if newText.hasPrefix(self.latestText) {
|
||||
self.latestText = newText
|
||||
} else if self.latestText.hasPrefix(newText) {
|
||||
// Keep the longer cached text.
|
||||
} else {
|
||||
self.latestText += newText
|
||||
}
|
||||
if self.spokenOffset > self.latestText.count {
|
||||
self.spokenOffset = self.latestText.count
|
||||
}
|
||||
}
|
||||
|
||||
private mutating func extractSegments(isFinal: Bool) -> [String] {
|
||||
let chars = Array(self.latestText)
|
||||
guard self.spokenOffset < chars.count else { return [] }
|
||||
var idx = self.spokenOffset
|
||||
var lastBoundary: Int?
|
||||
var inCodeBlock = self.inCodeBlock
|
||||
var buffer = ""
|
||||
var bufferAtBoundary = ""
|
||||
var inCodeBlockAtBoundary = inCodeBlock
|
||||
|
||||
while idx < chars.count {
|
||||
if idx + 2 < chars.count,
|
||||
chars[idx] == "`",
|
||||
chars[idx + 1] == "`",
|
||||
chars[idx + 2] == "`"
|
||||
{
|
||||
inCodeBlock.toggle()
|
||||
idx += 3
|
||||
continue
|
||||
}
|
||||
|
||||
if !inCodeBlock {
|
||||
buffer.append(chars[idx])
|
||||
if Self.isBoundary(chars[idx]) {
|
||||
lastBoundary = idx + 1
|
||||
bufferAtBoundary = buffer
|
||||
inCodeBlockAtBoundary = inCodeBlock
|
||||
}
|
||||
}
|
||||
|
||||
idx += 1
|
||||
}
|
||||
|
||||
if let boundary = lastBoundary {
|
||||
self.spokenOffset = boundary
|
||||
self.inCodeBlock = inCodeBlockAtBoundary
|
||||
let trimmed = bufferAtBoundary.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return trimmed.isEmpty ? [] : [trimmed]
|
||||
}
|
||||
|
||||
guard isFinal else { return [] }
|
||||
self.spokenOffset = chars.count
|
||||
self.inCodeBlock = inCodeBlock
|
||||
let trimmed = buffer.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return trimmed.isEmpty ? [] : [trimmed]
|
||||
}
|
||||
|
||||
private static func isBoundary(_ ch: Character) -> Bool {
|
||||
ch == "." || ch == "!" || ch == "?" || ch == "\n"
|
||||
}
|
||||
}
|
||||
|
||||
private struct IncrementalSpeechContext {
|
||||
let apiKey: String?
|
||||
let voiceId: String?
|
||||
let modelId: String?
|
||||
let outputFormat: String?
|
||||
let language: String?
|
||||
let directive: TalkDirective?
|
||||
let canUseElevenLabs: Bool
|
||||
}
|
||||
|
||||
#if DEBUG
|
||||
extension TalkModeManager {
|
||||
func _test_seedTranscript(_ transcript: String) {
|
||||
@@ -1017,6 +1426,10 @@ extension TalkModeManager {
|
||||
self.lastHeard = Date()
|
||||
}
|
||||
|
||||
func _test_handleTranscript(_ transcript: String, isFinal: Bool) async {
|
||||
await self.handleTranscript(transcript: transcript, isFinal: isFinal)
|
||||
}
|
||||
|
||||
func _test_backdateLastHeard(seconds: TimeInterval) {
|
||||
self.lastHeard = Date().addingTimeInterval(-seconds)
|
||||
}
|
||||
@@ -1024,5 +1437,13 @@ extension TalkModeManager {
|
||||
func _test_runSilenceCheck() async {
|
||||
await self.checkSilence()
|
||||
}
|
||||
|
||||
func _test_incrementalReset() {
|
||||
self.incrementalSpeechBuffer = IncrementalSpeechBuffer()
|
||||
}
|
||||
|
||||
func _test_incrementalIngest(_ text: String, isFinal: Bool) -> [String] {
|
||||
self.incrementalSpeechBuffer.ingest(text: text, isFinal: isFinal)
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -2,7 +2,9 @@ Sources/Gateway/GatewayConnectionController.swift
|
||||
Sources/Gateway/GatewayDiscoveryDebugLogView.swift
|
||||
Sources/Gateway/GatewayDiscoveryModel.swift
|
||||
Sources/Gateway/GatewaySettingsStore.swift
|
||||
Sources/Gateway/GatewayHealthMonitor.swift
|
||||
Sources/Gateway/KeychainStore.swift
|
||||
Sources/Capabilities/NodeCapabilityRouter.swift
|
||||
Sources/Camera/CameraController.swift
|
||||
Sources/Chat/ChatSheet.swift
|
||||
Sources/Chat/IOSGatewayChatTransport.swift
|
||||
@@ -58,6 +60,7 @@ Sources/Voice/VoiceWakePreferences.swift
|
||||
../shared/OpenClawKit/Sources/OpenClawKit/CanvasCommandParams.swift
|
||||
../shared/OpenClawKit/Sources/OpenClawKit/CanvasCommands.swift
|
||||
../shared/OpenClawKit/Sources/OpenClawKit/Capabilities.swift
|
||||
../shared/OpenClawKit/Sources/OpenClawKit/ChatCommands.swift
|
||||
../shared/OpenClawKit/Sources/OpenClawKit/ContactsCommands.swift
|
||||
../shared/OpenClawKit/Sources/OpenClawKit/DeviceCommands.swift
|
||||
../shared/OpenClawKit/Sources/OpenClawKit/OpenClawKitResources.swift
|
||||
|
||||
@@ -92,6 +92,7 @@ private func withUserDefaults<T>(_ updates: [String: Any?], _ body: () throws ->
|
||||
let commands = Set(controller._test_currentCommands())
|
||||
|
||||
#expect(commands.contains(OpenClawSystemCommand.notify.rawValue))
|
||||
#expect(commands.contains(OpenClawChatCommand.push.rawValue))
|
||||
#expect(!commands.contains(OpenClawSystemCommand.run.rawValue))
|
||||
#expect(!commands.contains(OpenClawSystemCommand.which.rawValue))
|
||||
#expect(!commands.contains(OpenClawSystemCommand.execApprovalsGet.rawValue))
|
||||
|
||||
60
apps/ios/Tests/GatewayHealthMonitorTests.swift
Normal file
60
apps/ios/Tests/GatewayHealthMonitorTests.swift
Normal file
@@ -0,0 +1,60 @@
|
||||
import Foundation
|
||||
import Testing
|
||||
@testable import OpenClaw
|
||||
|
||||
private actor Counter {
|
||||
private var value = 0
|
||||
|
||||
func increment() {
|
||||
value += 1
|
||||
}
|
||||
|
||||
func get() -> Int {
|
||||
value
|
||||
}
|
||||
|
||||
func set(_ newValue: Int) {
|
||||
value = newValue
|
||||
}
|
||||
}
|
||||
|
||||
@Suite struct GatewayHealthMonitorTests {
|
||||
@Test @MainActor func triggersFailureAfterThreshold() async {
|
||||
let failureCount = Counter()
|
||||
let monitor = GatewayHealthMonitor(
|
||||
config: .init(intervalSeconds: 0.001, timeoutSeconds: 0.0, maxFailures: 2))
|
||||
|
||||
monitor.start(
|
||||
check: { false },
|
||||
onFailure: { _ in
|
||||
await failureCount.increment()
|
||||
await monitor.stop()
|
||||
})
|
||||
|
||||
try? await Task.sleep(nanoseconds: 60_000_000)
|
||||
#expect(await failureCount.get() == 1)
|
||||
}
|
||||
|
||||
@Test @MainActor func resetsFailuresAfterSuccess() async {
|
||||
let failureCount = Counter()
|
||||
let calls = Counter()
|
||||
let monitor = GatewayHealthMonitor(
|
||||
config: .init(intervalSeconds: 0.001, timeoutSeconds: 0.0, maxFailures: 2))
|
||||
|
||||
monitor.start(
|
||||
check: {
|
||||
await calls.increment()
|
||||
let callCount = await calls.get()
|
||||
if callCount >= 6 {
|
||||
await monitor.stop()
|
||||
}
|
||||
return callCount % 2 == 0
|
||||
},
|
||||
onFailure: { _ in
|
||||
await failureCount.increment()
|
||||
})
|
||||
|
||||
try? await Task.sleep(nanoseconds: 60_000_000)
|
||||
#expect(await failureCount.get() == 0)
|
||||
}
|
||||
}
|
||||
@@ -448,6 +448,91 @@ private func decodePayload<T: Decodable>(_ json: String?, as type: T.Type) throw
|
||||
#expect(request.content.body == "World")
|
||||
}
|
||||
|
||||
@Test @MainActor func handleInvokeChatPushCreatesNotification() async throws {
|
||||
let notifier = TestNotificationCenter(status: .authorized)
|
||||
let deviceStatus = TestDeviceStatusService(
|
||||
statusPayload: OpenClawDeviceStatusPayload(
|
||||
battery: OpenClawBatteryStatusPayload(level: 0.5, state: .charging, lowPowerModeEnabled: false),
|
||||
thermal: OpenClawThermalStatusPayload(state: .nominal),
|
||||
storage: OpenClawStorageStatusPayload(totalBytes: 100, freeBytes: 50, usedBytes: 50),
|
||||
network: OpenClawNetworkStatusPayload(
|
||||
status: .satisfied,
|
||||
isExpensive: false,
|
||||
isConstrained: false,
|
||||
interfaces: [.wifi]),
|
||||
uptimeSeconds: 10),
|
||||
infoPayload: OpenClawDeviceInfoPayload(
|
||||
deviceName: "Test",
|
||||
modelIdentifier: "Test1,1",
|
||||
systemName: "iOS",
|
||||
systemVersion: "1.0",
|
||||
appVersion: "dev",
|
||||
appBuild: "0",
|
||||
locale: "en-US"))
|
||||
let emptyContact = OpenClawContactPayload(
|
||||
identifier: "c0",
|
||||
displayName: "",
|
||||
givenName: "",
|
||||
familyName: "",
|
||||
organizationName: "",
|
||||
phoneNumbers: [],
|
||||
emails: [])
|
||||
let emptyEvent = OpenClawCalendarEventPayload(
|
||||
identifier: "e0",
|
||||
title: "Test",
|
||||
startISO: "2024-01-01T00:00:00Z",
|
||||
endISO: "2024-01-01T00:30:00Z",
|
||||
isAllDay: false,
|
||||
location: nil,
|
||||
calendarTitle: nil)
|
||||
let emptyReminder = OpenClawReminderPayload(
|
||||
identifier: "r0",
|
||||
title: "Test",
|
||||
dueISO: nil,
|
||||
completed: false,
|
||||
listName: nil)
|
||||
let appModel = makeTestAppModel(
|
||||
notificationCenter: notifier,
|
||||
deviceStatusService: deviceStatus,
|
||||
photosService: TestPhotosService(payload: OpenClawPhotosLatestPayload(photos: [])),
|
||||
contactsService: TestContactsService(
|
||||
searchPayload: OpenClawContactsSearchPayload(contacts: []),
|
||||
addPayload: OpenClawContactsAddPayload(contact: emptyContact)),
|
||||
calendarService: TestCalendarService(
|
||||
eventsPayload: OpenClawCalendarEventsPayload(events: []),
|
||||
addPayload: OpenClawCalendarAddPayload(event: emptyEvent)),
|
||||
remindersService: TestRemindersService(
|
||||
listPayload: OpenClawRemindersListPayload(reminders: []),
|
||||
addPayload: OpenClawRemindersAddPayload(reminder: emptyReminder)),
|
||||
motionService: TestMotionService(
|
||||
activityPayload: OpenClawMotionActivityPayload(activities: []),
|
||||
pedometerPayload: OpenClawPedometerPayload(
|
||||
startISO: "2024-01-01T00:00:00Z",
|
||||
endISO: "2024-01-01T01:00:00Z",
|
||||
steps: nil,
|
||||
distanceMeters: nil,
|
||||
floorsAscended: nil,
|
||||
floorsDescended: nil)))
|
||||
|
||||
let params = OpenClawChatPushParams(text: "Ping", speak: false)
|
||||
let data = try JSONEncoder().encode(params)
|
||||
let json = String(decoding: data, as: UTF8.self)
|
||||
let req = BridgeInvokeRequest(
|
||||
id: "chat-push",
|
||||
command: OpenClawChatCommand.push.rawValue,
|
||||
paramsJSON: json)
|
||||
let res = await appModel._test_handleInvoke(req)
|
||||
#expect(res.ok == true)
|
||||
#expect(notifier.addedRequests.count == 1)
|
||||
let request = try #require(notifier.addedRequests.first)
|
||||
#expect(request.content.title == "OpenClaw")
|
||||
#expect(request.content.body == "Ping")
|
||||
let payloadJSON = try #require(res.payloadJSON)
|
||||
let decoded = try JSONDecoder().decode(OpenClawChatPushPayload.self, from: Data(payloadJSON.utf8))
|
||||
#expect((decoded.messageId ?? "").isEmpty == false)
|
||||
#expect(request.identifier == decoded.messageId)
|
||||
}
|
||||
|
||||
@Test @MainActor func handleInvokeDeviceAndDataCommandsReturnPayloads() async throws {
|
||||
let deviceStatusPayload = OpenClawDeviceStatusPayload(
|
||||
battery: OpenClawBatteryStatusPayload(level: 0.25, state: .unplugged, lowPowerModeEnabled: false),
|
||||
@@ -723,6 +808,28 @@ private func decodePayload<T: Decodable>(_ json: String?, as type: T.Type) throw
|
||||
#expect(oncePayload.status == "offline")
|
||||
}
|
||||
|
||||
@Test @MainActor func handleInvokePushToTalkOnceStopsOnFinalTranscript() async throws {
|
||||
let talkMode = TalkModeManager(allowSimulatorCapture: true)
|
||||
talkMode.updateGatewayConnected(false)
|
||||
let appModel = makeTalkTestAppModel(talkMode: talkMode)
|
||||
|
||||
let onceReq = BridgeInvokeRequest(id: "ptt-once-final", command: OpenClawTalkCommand.pttOnce.rawValue)
|
||||
let onceTask = Task { await appModel._test_handleInvoke(onceReq) }
|
||||
|
||||
for _ in 0..<5 where !talkMode.isPushToTalkActive {
|
||||
await Task.yield()
|
||||
}
|
||||
#expect(talkMode.isPushToTalkActive == true)
|
||||
|
||||
await talkMode._test_handleTranscript("Hello final", isFinal: true)
|
||||
|
||||
let onceRes = await onceTask.value
|
||||
#expect(onceRes.ok == true)
|
||||
let oncePayload = try decodePayload(onceRes.payloadJSON, as: OpenClawTalkPTTStopPayload.self)
|
||||
#expect(oncePayload.transcript == "Hello final")
|
||||
#expect(oncePayload.status == "offline")
|
||||
}
|
||||
|
||||
@Test @MainActor func handleDeepLinkSetsErrorWhenNotConnected() async {
|
||||
let appModel = NodeAppModel()
|
||||
let url = URL(string: "openclaw://agent?message=hello")!
|
||||
|
||||
33
apps/ios/Tests/TalkModeIncrementalTests.swift
Normal file
33
apps/ios/Tests/TalkModeIncrementalTests.swift
Normal file
@@ -0,0 +1,33 @@
|
||||
import Foundation
|
||||
import Testing
|
||||
@testable import OpenClaw
|
||||
|
||||
@Suite struct TalkModeIncrementalTests {
|
||||
@Test @MainActor func incrementalSpeechSplitsOnBoundary() {
|
||||
let talkMode = TalkModeManager(allowSimulatorCapture: true)
|
||||
talkMode._test_incrementalReset()
|
||||
let segments = talkMode._test_incrementalIngest("Hello world. Next", isFinal: false)
|
||||
#expect(segments.count == 1)
|
||||
#expect(segments.first == "Hello world.")
|
||||
}
|
||||
|
||||
@Test @MainActor func incrementalSpeechSkipsDirectiveLine() {
|
||||
let talkMode = TalkModeManager(allowSimulatorCapture: true)
|
||||
talkMode._test_incrementalReset()
|
||||
let segments = talkMode._test_incrementalIngest("{\"voice\":\"abc\"}\nHello.", isFinal: false)
|
||||
#expect(segments.count == 1)
|
||||
#expect(segments.first == "Hello.")
|
||||
}
|
||||
|
||||
@Test @MainActor func incrementalSpeechIgnoresCodeBlocks() {
|
||||
let talkMode = TalkModeManager(allowSimulatorCapture: true)
|
||||
talkMode._test_incrementalReset()
|
||||
let text = "Here is code:\n```js\nx=1\n```\nDone."
|
||||
let segments = talkMode._test_incrementalIngest(text, isFinal: true)
|
||||
#expect(segments.count == 1)
|
||||
let value = segments.first ?? ""
|
||||
#expect(value.contains("x=1") == false)
|
||||
#expect(value.contains("Here is code") == true)
|
||||
#expect(value.contains("Done.") == true)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import Foundation
|
||||
|
||||
public enum OpenClawChatCommand: String, Codable, Sendable {
|
||||
case push = "chat.push"
|
||||
}
|
||||
|
||||
public struct OpenClawChatPushParams: Codable, Sendable, Equatable {
|
||||
public var text: String
|
||||
public var speak: Bool?
|
||||
|
||||
public init(text: String, speak: Bool? = nil) {
|
||||
self.text = text
|
||||
self.speak = speak
|
||||
}
|
||||
}
|
||||
|
||||
public struct OpenClawChatPushPayload: Codable, Sendable, Equatable {
|
||||
public var messageId: String?
|
||||
|
||||
public init(messageId: String? = nil) {
|
||||
self.messageId = messageId
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,8 @@ const MOTION_COMMANDS = ["motion.activity", "motion.pedometer"];
|
||||
|
||||
const SYSTEM_NOTIFY_COMMANDS = ["system.notify"];
|
||||
|
||||
const CHAT_COMMANDS = ["chat.push"];
|
||||
|
||||
const TALK_COMMANDS = ["talk.ptt.start", "talk.ptt.stop", "talk.ptt.cancel", "talk.ptt.once"];
|
||||
|
||||
const SYSTEM_COMMANDS = [
|
||||
@@ -52,6 +54,7 @@ const PLATFORM_DEFAULTS: Record<string, string[]> = {
|
||||
...SCREEN_COMMANDS,
|
||||
...LOCATION_COMMANDS,
|
||||
...SYSTEM_NOTIFY_COMMANDS,
|
||||
...CHAT_COMMANDS,
|
||||
...DEVICE_COMMANDS,
|
||||
...PHOTOS_COMMANDS,
|
||||
...CONTACTS_COMMANDS,
|
||||
|
||||
Reference in New Issue
Block a user