diff --git a/apps/ios/Sources/Gateway/GatewayConnectionController.swift b/apps/ios/Sources/Gateway/GatewayConnectionController.swift index 65d099c010..6f7b187cc7 100644 --- a/apps/ios/Sources/Gateway/GatewayConnectionController.swift +++ b/apps/ios/Sources/Gateway/GatewayConnectionController.swift @@ -1,14 +1,39 @@ import OpenClawKit +import AVFoundation +import CoreLocation import Darwin import Foundation import Network import Observation +import ReplayKit import SwiftUI import UIKit @MainActor @Observable final class GatewayConnectionController { + struct PermissionStatusProvider: Sendable { + var cameraStatus: @Sendable () -> AVAuthorizationStatus + var microphoneStatus: @Sendable () -> AVAuthorizationStatus + var locationStatus: @Sendable () -> CLAuthorizationStatus + var locationServicesEnabled: @Sendable () -> Bool + var screenRecordingAvailable: @Sendable () -> Bool + + static func live() -> PermissionStatusProvider { + PermissionStatusProvider( + cameraStatus: { AVCaptureDevice.authorizationStatus(for: .video) }, + microphoneStatus: { AVCaptureDevice.authorizationStatus(for: .audio) }, + locationStatus: { + if #available(iOS 14.0, *) { + return CLLocationManager.authorizationStatus() + } + return CLLocationManager().authorizationStatus + }, + locationServicesEnabled: { CLLocationManager.locationServicesEnabled() }, + screenRecordingAvailable: { RPScreenRecorder.shared().isAvailable }) + } + } + private(set) var gateways: [GatewayDiscoveryModel.DiscoveredGateway] = [] private(set) var discoveryStatusText: String = "Idle" private(set) var discoveryDebugLog: [GatewayDiscoveryModel.DebugLogEntry] = [] @@ -16,9 +41,15 @@ final class GatewayConnectionController { private let discovery = GatewayDiscoveryModel() private weak var appModel: NodeAppModel? private var didAutoConnect = false + private let permissionProvider: PermissionStatusProvider - init(appModel: NodeAppModel, startDiscovery: Bool = true) { + init( + appModel: NodeAppModel, + startDiscovery: Bool = true, + permissionProvider: PermissionStatusProvider = PermissionStatusProvider.live()) + { self.appModel = appModel + self.permissionProvider = permissionProvider GatewaySettingsStore.bootstrapPersistence() let defaults = UserDefaults.standard @@ -282,7 +313,7 @@ final class GatewayConnectionController { scopes: [], caps: self.currentCaps(), commands: self.currentCommands(), - permissions: [:], + permissions: self.currentPermissions(), clientId: "openclaw-ios", clientMode: "node", clientDisplayName: displayName) @@ -335,10 +366,6 @@ final class GatewayConnectionController { OpenClawCanvasA2UICommand.reset.rawValue, OpenClawScreenCommand.record.rawValue, OpenClawSystemCommand.notify.rawValue, - OpenClawSystemCommand.which.rawValue, - OpenClawSystemCommand.run.rawValue, - OpenClawSystemCommand.execApprovalsGet.rawValue, - OpenClawSystemCommand.execApprovalsSet.rawValue, ] let caps = Set(self.currentCaps()) @@ -354,6 +381,32 @@ final class GatewayConnectionController { return commands } + private func currentPermissions() -> [String: Bool] { + let camera = self.permissionProvider.cameraStatus() + let microphone = self.permissionProvider.microphoneStatus() + let locationStatus = self.permissionProvider.locationStatus() + let locationEnabled = self.permissionProvider.locationServicesEnabled() + let screenRecordingAvailable = self.permissionProvider.screenRecordingAvailable() + + return [ + "camera": camera == .authorized, + "microphone": microphone == .authorized, + "location": locationEnabled && Self.isLocationAuthorized(status: locationStatus), + "screenRecording": screenRecordingAvailable, + ] + } + + private static func isLocationAuthorized(status: CLAuthorizationStatus) -> Bool { + switch status { + case .authorizedAlways, .authorizedWhenInUse: + return true + case .authorized: + return true + default: + return false + } + } + private func platformString() -> String { let v = ProcessInfo.processInfo.operatingSystemVersion let name = switch UIDevice.current.userInterfaceIdiom { @@ -407,6 +460,10 @@ extension GatewayConnectionController { self.currentCommands() } + func _test_currentPermissions() -> [String: Bool] { + self.currentPermissions() + } + func _test_platformString() -> String { self.platformString() } diff --git a/apps/ios/Sources/Model/NodeAppModel.swift b/apps/ios/Sources/Model/NodeAppModel.swift index 963318a8a2..54858875b7 100644 --- a/apps/ios/Sources/Model/NodeAppModel.swift +++ b/apps/ios/Sources/Model/NodeAppModel.swift @@ -3,6 +3,63 @@ import Network import Observation import SwiftUI import UIKit +import UserNotifications + +enum NotificationAuthorizationStatus: Sendable { + case notDetermined + case denied + case authorized + case provisional + case ephemeral +} + +protocol NotificationCentering: Sendable { + func authorizationStatus() async -> NotificationAuthorizationStatus + func requestAuthorization(options: UNAuthorizationOptions) async throws -> Bool + func add(_ request: UNNotificationRequest) async throws +} + +struct LiveNotificationCenter: NotificationCentering, @unchecked Sendable { + private let center: UNUserNotificationCenter + + init(center: UNUserNotificationCenter = .current()) { + self.center = center + } + + func authorizationStatus() async -> NotificationAuthorizationStatus { + let settings = await self.center.notificationSettings() + return switch settings.authorizationStatus { + case .authorized: + .authorized + case .provisional: + .provisional + case .ephemeral: + .ephemeral + case .denied: + .denied + case .notDetermined: + .notDetermined + @unknown default: + .denied + } + } + + func requestAuthorization(options: UNAuthorizationOptions) async throws -> Bool { + try await self.center.requestAuthorization(options: options) + } + + func add(_ request: UNNotificationRequest) async throws { + try await withCheckedThrowingContinuation { cont in + self.center.add(request) { error in + if let error { + cont.resume(throwing: error) + } else { + cont.resume() + } + } + } + } +} @MainActor @Observable @@ -28,6 +85,7 @@ final class NodeAppModel { private let gateway = GatewayNodeSession() private var gatewayTask: Task? private var voiceWakeSyncTask: Task? + private let notificationCenter: NotificationCentering @ObservationIgnored private var cameraHUDDismissTask: Task? let voiceWake = VoiceWakeManager() let talkMode = TalkModeManager() @@ -42,7 +100,8 @@ final class NodeAppModel { var cameraFlashNonce: Int = 0 var screenRecordActive: Bool = false - init() { + init(notificationCenter: NotificationCentering = LiveNotificationCenter()) { + self.notificationCenter = notificationCenter self.voiceWake.configure { [weak self] cmd in guard let self else { return } let sessionKey = await MainActor.run { self.mainSessionKey } @@ -542,12 +601,14 @@ final class NodeAppModel { return try await self.handleCameraInvoke(req) case OpenClawScreenCommand.record.rawValue: return try await self.handleScreenRecordInvoke(req) + case OpenClawSystemCommand.notify.rawValue: + return try await self.handleSystemNotify(req) default: return BridgeInvokeResponse( id: req.id, ok: false, error: OpenClawNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command")) - } + } } catch { if command.hasPrefix("camera.") { let text = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription @@ -628,6 +689,7 @@ final class NodeAppModel { case OpenClawCanvasCommand.present.rawValue: let params = (try? Self.decodeParams(OpenClawCanvasPresentParams.self, from: req.paramsJSON)) ?? OpenClawCanvasPresentParams() + // iOS ignores placement params (canvas presents full-screen). let url = params.url?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" if url.isEmpty { self.screen.showDefaultCanvas() @@ -636,6 +698,7 @@ final class NodeAppModel { } return BridgeInvokeResponse(id: req.id, ok: true) case OpenClawCanvasCommand.hide.rawValue: + self.showLocalCanvasOnDisconnect() return BridgeInvokeResponse(id: req.id, ok: true) case OpenClawCanvasCommand.navigate.rawValue: let params = try Self.decodeParams(OpenClawCanvasNavigateParams.self, from: req.paramsJSON) @@ -859,6 +922,58 @@ final class NodeAppModel { return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) } + private func handleSystemNotify(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let params = try Self.decodeParams(OpenClawSystemNotifyParams.self, from: req.paramsJSON) + let status = await self.notificationCenter.authorizationStatus() + let authorized: Bool + switch status { + case .authorized, .provisional, .ephemeral: + authorized = true + case .notDetermined: + authorized = (try await self.notificationCenter + .requestAuthorization(options: [.alert, .sound, .badge])) + case .denied: + authorized = false + } + + guard authorized else { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: OpenClawNodeError( + code: .unavailable, + message: "NOTIFICATION_PERMISSION_REQUIRED: enable Notifications in Settings")) + } + + let content = UNMutableNotificationContent() + content.title = params.title + content.body = params.body + let sound = params.sound?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if sound.isEmpty { + content.sound = .default + } else { + content.sound = UNNotificationSound(named: UNNotificationSoundName(rawValue: sound)) + } + if let priority = params.priority { + switch priority { + case .passive: + content.interruptionLevel = .passive + case .active: + content.interruptionLevel = .active + case .timeSensitive: + content.interruptionLevel = .timeSensitive + } + } + + let trigger = UNTimeIntervalNotificationTrigger(timeInterval: 0.1, repeats: false) + let request = UNNotificationRequest( + identifier: UUID().uuidString, + content: content, + trigger: trigger) + try await self.notificationCenter.add(request) + return BridgeInvokeResponse(id: req.id, ok: true) + } + } private extension NodeAppModel { diff --git a/apps/ios/Tests/GatewayConnectionControllerTests.swift b/apps/ios/Tests/GatewayConnectionControllerTests.swift index 0d3bdbba0e..0ac7718d2c 100644 --- a/apps/ios/Tests/GatewayConnectionControllerTests.swift +++ b/apps/ios/Tests/GatewayConnectionControllerTests.swift @@ -1,4 +1,6 @@ import OpenClawKit +import AVFoundation +import CoreLocation import Foundation import Testing import UIKit @@ -76,4 +78,41 @@ private func withUserDefaults(_ updates: [String: Any?], _ body: () throws -> #expect(commands.contains(OpenClawLocationCommand.get.rawValue)) } } + + @Test @MainActor func currentCommandsExcludeSystemExecButKeepNotify() { + withUserDefaults([ + "node.instanceId": "ios-test", + ]) { + let appModel = NodeAppModel() + let controller = GatewayConnectionController(appModel: appModel, startDiscovery: false) + let commands = Set(controller._test_currentCommands()) + + #expect(commands.contains(OpenClawSystemCommand.notify.rawValue)) + #expect(commands.contains(OpenClawSystemCommand.run.rawValue) == false) + #expect(commands.contains(OpenClawSystemCommand.which.rawValue) == false) + #expect(commands.contains(OpenClawSystemCommand.execApprovalsGet.rawValue) == false) + #expect(commands.contains(OpenClawSystemCommand.execApprovalsSet.rawValue) == false) + } + } + + @Test @MainActor func currentPermissionsIncludeExpectedKeys() { + let provider = GatewayConnectionController.PermissionStatusProvider( + cameraStatus: { .authorized }, + microphoneStatus: { .denied }, + locationStatus: { .authorizedWhenInUse }, + locationServicesEnabled: { true }, + screenRecordingAvailable: { false }) + + let appModel = NodeAppModel() + let controller = GatewayConnectionController( + appModel: appModel, + startDiscovery: false, + permissionProvider: provider) + let permissions = controller._test_currentPermissions() + + #expect(permissions["camera"] == true) + #expect(permissions["microphone"] == false) + #expect(permissions["location"] == true) + #expect(permissions["screenRecording"] == false) + } } diff --git a/apps/ios/Tests/NodeAppModelInvokeTests.swift b/apps/ios/Tests/NodeAppModelInvokeTests.swift index 124059021d..b87cd5afd3 100644 --- a/apps/ios/Tests/NodeAppModelInvokeTests.swift +++ b/apps/ios/Tests/NodeAppModelInvokeTests.swift @@ -124,6 +124,11 @@ private func withUserDefaults(_ updates: [String: Any?], _ body: () throws -> let payloadData = try #require(evalRes.payloadJSON?.data(using: .utf8)) let payload = try JSONSerialization.jsonObject(with: payloadData) as? [String: Any] #expect(payload?["result"] as? String == "2") + + let hide = BridgeInvokeRequest(id: "hide", command: OpenClawCanvasCommand.hide.rawValue) + let hideRes = await appModel._test_handleInvoke(hide) + #expect(hideRes.ok == true) + #expect(appModel.screen.urlString.isEmpty) } @Test @MainActor func handleInvokeA2UICommandsFailWhenHostMissing() async throws { diff --git a/apps/ios/Tests/NodeAppModelNotifyTests.swift b/apps/ios/Tests/NodeAppModelNotifyTests.swift new file mode 100644 index 0000000000..4d24c8d5cd --- /dev/null +++ b/apps/ios/Tests/NodeAppModelNotifyTests.swift @@ -0,0 +1,66 @@ +import OpenClawKit +import Foundation +import Testing +import UserNotifications +@testable import OpenClaw + +actor TestNotificationCenter: NotificationCentering { + private var status: NotificationAuthorizationStatus + private let requestResult: Bool + private var requestedAuthorization: Bool = false + private var storedRequests: [UNNotificationRequest] = [] + + init(status: NotificationAuthorizationStatus, requestResult: Bool) { + self.status = status + self.requestResult = requestResult + } + + func authorizationStatus() async -> NotificationAuthorizationStatus { + status + } + + func requestAuthorization(options _: UNAuthorizationOptions) async throws -> Bool { + self.requestedAuthorization = true + if self.requestResult { + self.status = .authorized + } + return self.requestResult + } + + func add(_ request: UNNotificationRequest) async throws { + self.storedRequests.append(request) + } + + func didRequestAuthorization() async -> Bool { + requestedAuthorization + } + + func requests() async -> [UNNotificationRequest] { + storedRequests + } +} + +@Suite(.serialized) struct NodeAppModelNotifyTests { + @Test @MainActor func handleSystemNotifyRequestsPermissionAndAddsNotification() async throws { + let center = TestNotificationCenter(status: .notDetermined, requestResult: true) + let appModel = NodeAppModel(notificationCenter: center) + + let params = OpenClawSystemNotifyParams(title: "Hello", body: "World") + let data = try JSONEncoder().encode(params) + let json = String(decoding: data, as: UTF8.self) + + let req = BridgeInvokeRequest( + id: "notify", + command: OpenClawSystemCommand.notify.rawValue, + paramsJSON: json) + + let res = await appModel._test_handleInvoke(req) + #expect(res.ok == true) + #expect(await center.didRequestAuthorization() == true) + + let requests = await center.requests() + #expect(requests.count == 1) + #expect(requests.first?.content.title == "Hello") + #expect(requests.first?.content.body == "World") + } +}