diff --git a/CHANGELOG.md b/CHANGELOG.md index 18e0f96cd0..b0dcf2b6d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- iOS/Screen: move `WKWebView` lifecycle ownership into `ScreenWebView` coordinator and explicit attach/detach flow to reduce gesture/lifecycle crash risk (`__NSArrayM insertObject:atIndex:` paths) during screen tab updates. (#20366) Thanks @ngutman. - Protocol/Apple: regenerate Swift gateway models for `push.test` so `pnpm protocol:check` stays green on main. Thanks @mbelinky. - Canvas/A2UI: improve bundled-asset resolution and empty-state handling so UI fallbacks render reliably. (#20312) Thanks @mbelinky. - UI/Sessions: accept the canonical main session-key alias in Chat UI flows so main-session routing stays consistent. (#20311) Thanks @mbelinky. diff --git a/apps/ios/Sources/Model/NodeAppModel.swift b/apps/ios/Sources/Model/NodeAppModel.swift index 6b8885dfcb..1d09251dd7 100644 --- a/apps/ios/Sources/Model/NodeAppModel.swift +++ b/apps/ios/Sources/Model/NodeAppModel.swift @@ -73,26 +73,6 @@ final class NodeAppModel { var lastShareEventText: String = "No share events yet." var openChatRequestID: Int = 0 - var mainSessionKey: String { - let base = SessionKey.normalizeMainKey(self.mainSessionBaseKey) - let agentId = (self.selectedAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines) - let defaultId = (self.gatewayDefaultAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines) - if agentId.isEmpty || (!defaultId.isEmpty && agentId == defaultId) { return base } - return SessionKey.makeAgentSessionKey(agentId: agentId, baseKey: base) - } - - var activeAgentName: String { - let agentId = (self.selectedAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines) - let defaultId = (self.gatewayDefaultAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines) - let resolvedId = agentId.isEmpty ? defaultId : agentId - if resolvedId.isEmpty { return "Main" } - if let match = self.gatewayAgents.first(where: { $0.id == resolvedId }) { - let name = (match.name ?? "").trimmingCharacters(in: .whitespacesAndNewlines) - return name.isEmpty ? match.id : name - } - return resolvedId - } - // Primary "node" connection: used for device capabilities and node.invoke requests. private let nodeGateway = GatewayNodeSession() // Secondary "operator" connection: used for chat/talk/config/voicewake requests. @@ -1615,6 +1595,26 @@ private extension NodeAppModel { } extension NodeAppModel { + var mainSessionKey: String { + let base = SessionKey.normalizeMainKey(self.mainSessionBaseKey) + let agentId = (self.selectedAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + let defaultId = (self.gatewayDefaultAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + if agentId.isEmpty || (!defaultId.isEmpty && agentId == defaultId) { return base } + return SessionKey.makeAgentSessionKey(agentId: agentId, baseKey: base) + } + + var activeAgentName: String { + let agentId = (self.selectedAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + let defaultId = (self.gatewayDefaultAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + let resolvedId = agentId.isEmpty ? defaultId : agentId + if resolvedId.isEmpty { return "Main" } + if let match = self.gatewayAgents.first(where: { $0.id == resolvedId }) { + let name = (match.name ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + return name.isEmpty ? match.id : name + } + return resolvedId + } + func connectToGateway( url: URL, gatewayStableID: String, diff --git a/apps/ios/Sources/Screen/ScreenController.swift b/apps/ios/Sources/Screen/ScreenController.swift index 506b78a230..0045232362 100644 --- a/apps/ios/Sources/Screen/ScreenController.swift +++ b/apps/ios/Sources/Screen/ScreenController.swift @@ -1,14 +1,12 @@ import OpenClawKit import Observation -import SwiftUI +import UIKit import WebKit @MainActor @Observable final class ScreenController { - let webView: WKWebView - private let navigationDelegate: ScreenNavigationDelegate - private let a2uiActionHandler: CanvasA2UIActionMessageHandler + private weak var activeWebView: WKWebView? var urlString: String = "" var errorText: String? @@ -24,29 +22,6 @@ final class ScreenController { private var debugStatusSubtitle: String? init() { - let config = WKWebViewConfiguration() - config.websiteDataStore = .nonPersistent() - let a2uiActionHandler = CanvasA2UIActionMessageHandler() - let userContentController = WKUserContentController() - for name in CanvasA2UIActionMessageHandler.handlerNames { - userContentController.add(a2uiActionHandler, name: name) - } - config.userContentController = userContentController - self.navigationDelegate = ScreenNavigationDelegate() - self.a2uiActionHandler = a2uiActionHandler - self.webView = WKWebView(frame: .zero, configuration: config) - // Canvas scaffold is a fully self-contained HTML page; avoid relying on transparency underlays. - self.webView.isOpaque = true - self.webView.backgroundColor = .black - self.webView.scrollView.backgroundColor = .black - self.webView.scrollView.contentInsetAdjustmentBehavior = .never - self.webView.scrollView.contentInset = .zero - self.webView.scrollView.scrollIndicatorInsets = .zero - self.webView.scrollView.automaticallyAdjustsScrollIndicatorInsets = false - self.applyScrollBehavior() - self.webView.navigationDelegate = self.navigationDelegate - self.navigationDelegate.controller = self - a2uiActionHandler.controller = self self.reload() } @@ -71,24 +46,26 @@ final class ScreenController { } func reload() { - let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines) self.applyScrollBehavior() + guard let webView = self.activeWebView else { return } + + let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines) if trimmed.isEmpty { guard let url = Self.canvasScaffoldURL else { return } self.errorText = nil - self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent()) + webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent()) return + } + + guard let url = URL(string: trimmed) else { + self.errorText = "Invalid URL: \(trimmed)" + return + } + self.errorText = nil + if url.isFileURL { + webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent()) } else { - guard let url = URL(string: trimmed) else { - self.errorText = "Invalid URL: \(trimmed)" - return - } - self.errorText = nil - if url.isFileURL { - self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent()) - } else { - self.webView.load(URLRequest(url: url)) - } + webView.load(URLRequest(url: url)) } } @@ -108,7 +85,8 @@ final class ScreenController { self.applyDebugStatusIfNeeded() } - fileprivate func applyDebugStatusIfNeeded() { + func applyDebugStatusIfNeeded() { + guard let webView = self.activeWebView else { return } let enabled = self.debugStatusEnabled let title = self.debugStatusTitle let subtitle = self.debugStatusSubtitle @@ -127,7 +105,7 @@ final class ScreenController { } catch (_) {} })() """ - self.webView.evaluateJavaScript(js) { _, _ in } + webView.evaluateJavaScript(js) { _, _ in } } func waitForA2UIReady(timeoutMs: Int) async -> Bool { @@ -154,8 +132,13 @@ final class ScreenController { } func eval(javaScript: String) async throws -> String { - try await withCheckedThrowingContinuation { cont in - self.webView.evaluateJavaScript(javaScript) { result, error in + guard let webView = self.activeWebView else { + throw NSError(domain: "Screen", code: 3, userInfo: [ + NSLocalizedDescriptionKey: "web view unavailable", + ]) + } + return try await withCheckedThrowingContinuation { cont in + webView.evaluateJavaScript(javaScript) { result, error in if let error { cont.resume(throwing: error) return @@ -174,8 +157,13 @@ final class ScreenController { if let maxWidth { config.snapshotWidth = NSNumber(value: Double(maxWidth)) } + guard let webView = self.activeWebView else { + throw NSError(domain: "Screen", code: 3, userInfo: [ + NSLocalizedDescriptionKey: "web view unavailable", + ]) + } let image: UIImage = try await withCheckedThrowingContinuation { cont in - self.webView.takeSnapshot(with: config) { image, error in + webView.takeSnapshot(with: config) { image, error in if let error { cont.resume(throwing: error) return @@ -206,8 +194,13 @@ final class ScreenController { if let maxWidth { config.snapshotWidth = NSNumber(value: Double(maxWidth)) } + guard let webView = self.activeWebView else { + throw NSError(domain: "Screen", code: 3, userInfo: [ + NSLocalizedDescriptionKey: "web view unavailable", + ]) + } let image: UIImage = try await withCheckedThrowingContinuation { cont in - self.webView.takeSnapshot(with: config) { image, error in + webView.takeSnapshot(with: config) { image, error in if let error { cont.resume(throwing: error) return @@ -238,6 +231,17 @@ final class ScreenController { return data.base64EncodedString() } + func attachWebView(_ webView: WKWebView) { + self.activeWebView = webView + self.reload() + self.applyDebugStatusIfNeeded() + } + + func detachWebView(_ webView: WKWebView) { + guard self.activeWebView === webView else { return } + self.activeWebView = nil + } + private static func bundledResourceURL( name: String, ext: String, @@ -277,9 +281,10 @@ final class ScreenController { } private func applyScrollBehavior() { + guard let webView = self.activeWebView else { return } let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines) let allowScroll = !trimmed.isEmpty - let scrollView = self.webView.scrollView + let scrollView = webView.scrollView // Default canvas needs raw touch events; external pages should scroll. scrollView.isScrollEnabled = allowScroll scrollView.bounces = allowScroll @@ -366,72 +371,3 @@ extension Double { return self } } - -// MARK: - Navigation Delegate - -/// Handles navigation policy to intercept openclaw:// deep links from canvas -@MainActor -private final class ScreenNavigationDelegate: NSObject, WKNavigationDelegate { - weak var controller: ScreenController? - - func webView( - _ webView: WKWebView, - decidePolicyFor navigationAction: WKNavigationAction, - decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void) - { - guard let url = navigationAction.request.url else { - decisionHandler(.allow) - return - } - - // Intercept openclaw:// deep links. - if url.scheme?.lowercased() == "openclaw" { - decisionHandler(.cancel) - self.controller?.onDeepLink?(url) - return - } - - decisionHandler(.allow) - } - - func webView( - _: WKWebView, - didFailProvisionalNavigation _: WKNavigation?, - withError error: any Error) - { - self.controller?.errorText = error.localizedDescription - } - - func webView(_: WKWebView, didFinish _: WKNavigation?) { - self.controller?.errorText = nil - self.controller?.applyDebugStatusIfNeeded() - } - - func webView(_: WKWebView, didFail _: WKNavigation?, withError error: any Error) { - self.controller?.errorText = error.localizedDescription - } -} - -private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler { - static let messageName = "openclawCanvasA2UIAction" - static let handlerNames = [messageName] - - weak var controller: ScreenController? - - func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) { - guard Self.handlerNames.contains(message.name) else { return } - guard let controller else { return } - - guard let url = message.webView?.url else { return } - if url.isFileURL { - guard controller.isTrustedCanvasUIURL(url) else { return } - } else { - // For security, only accept actions from local-network pages (e.g. the canvas host). - guard controller.isLocalNetworkCanvasURL(url) else { return } - } - - guard let body = ScreenController.parseA2UIActionBody(message.body) else { return } - - controller.onA2UIAction?(body) - } -} diff --git a/apps/ios/Sources/Screen/ScreenWebView.swift b/apps/ios/Sources/Screen/ScreenWebView.swift index c464521be5..a30d78cbd0 100644 --- a/apps/ios/Sources/Screen/ScreenWebView.swift +++ b/apps/ios/Sources/Screen/ScreenWebView.swift @@ -5,11 +5,189 @@ import WebKit struct ScreenWebView: UIViewRepresentable { var controller: ScreenController - func makeUIView(context: Context) -> WKWebView { - self.controller.webView + func makeCoordinator() -> ScreenWebViewCoordinator { + ScreenWebViewCoordinator(controller: self.controller) } - func updateUIView(_ webView: WKWebView, context: Context) { - // State changes are driven by ScreenController. + func makeUIView(context: Context) -> UIView { + context.coordinator.makeContainerView() + } + + func updateUIView(_: UIView, context: Context) { + context.coordinator.updateController(self.controller) + } + + static func dismantleUIView(_: UIView, coordinator: ScreenWebViewCoordinator) { + coordinator.teardown() + } +} + +@MainActor +final class ScreenWebViewCoordinator: NSObject { + private weak var controller: ScreenController? + private let navigationDelegate = ScreenNavigationDelegate() + private let a2uiActionHandler = CanvasA2UIActionMessageHandler() + private let userContentController = WKUserContentController() + + private(set) var managedWebView: WKWebView? + private weak var containerView: UIView? + + init(controller: ScreenController) { + self.controller = controller + super.init() + self.navigationDelegate.controller = controller + self.a2uiActionHandler.controller = controller + } + + func makeContainerView() -> UIView { + if let containerView { + return containerView + } + + let container = UIView(frame: .zero) + container.backgroundColor = .black + + let webView = Self.makeWebView(userContentController: self.userContentController) + webView.navigationDelegate = self.navigationDelegate + self.installA2UIHandlers() + + webView.translatesAutoresizingMaskIntoConstraints = false + container.addSubview(webView) + NSLayoutConstraint.activate([ + webView.leadingAnchor.constraint(equalTo: container.leadingAnchor), + webView.trailingAnchor.constraint(equalTo: container.trailingAnchor), + webView.topAnchor.constraint(equalTo: container.topAnchor), + webView.bottomAnchor.constraint(equalTo: container.bottomAnchor), + ]) + + self.managedWebView = webView + self.containerView = container + self.controller?.attachWebView(webView) + return container + } + + func updateController(_ controller: ScreenController) { + let previousController = self.controller + let controllerChanged = self.controller !== controller + self.controller = controller + self.navigationDelegate.controller = controller + self.a2uiActionHandler.controller = controller + if controllerChanged, let managedWebView { + previousController?.detachWebView(managedWebView) + controller.attachWebView(managedWebView) + } + } + + func teardown() { + if let managedWebView { + self.controller?.detachWebView(managedWebView) + managedWebView.navigationDelegate = nil + } + self.removeA2UIHandlers() + self.navigationDelegate.controller = nil + self.a2uiActionHandler.controller = nil + self.managedWebView = nil + self.containerView = nil + } + + private static func makeWebView(userContentController: WKUserContentController) -> WKWebView { + let config = WKWebViewConfiguration() + config.websiteDataStore = .nonPersistent() + config.userContentController = userContentController + + let webView = WKWebView(frame: .zero, configuration: config) + // Canvas scaffold is a fully self-contained HTML page; avoid relying on transparency underlays. + webView.isOpaque = true + webView.backgroundColor = .black + + let scrollView = webView.scrollView + scrollView.backgroundColor = .black + scrollView.contentInsetAdjustmentBehavior = .never + scrollView.contentInset = .zero + scrollView.scrollIndicatorInsets = .zero + scrollView.automaticallyAdjustsScrollIndicatorInsets = false + + return webView + } + + private func installA2UIHandlers() { + for name in CanvasA2UIActionMessageHandler.handlerNames { + self.userContentController.add(self.a2uiActionHandler, name: name) + } + } + + private func removeA2UIHandlers() { + for name in CanvasA2UIActionMessageHandler.handlerNames { + self.userContentController.removeScriptMessageHandler(forName: name) + } + } +} + +// MARK: - Navigation Delegate + +/// Handles navigation policy to intercept openclaw:// deep links from canvas +@MainActor +private final class ScreenNavigationDelegate: NSObject, WKNavigationDelegate { + weak var controller: ScreenController? + + func webView( + _: WKWebView, + decidePolicyFor navigationAction: WKNavigationAction, + decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void) + { + guard let url = navigationAction.request.url else { + decisionHandler(.allow) + return + } + + // Intercept openclaw:// deep links. + if url.scheme?.lowercased() == "openclaw" { + decisionHandler(.cancel) + self.controller?.onDeepLink?(url) + return + } + + decisionHandler(.allow) + } + + func webView( + _: WKWebView, + didFailProvisionalNavigation _: WKNavigation?, + withError error: any Error) + { + self.controller?.errorText = error.localizedDescription + } + + func webView(_: WKWebView, didFinish _: WKNavigation?) { + self.controller?.errorText = nil + self.controller?.applyDebugStatusIfNeeded() + } + + func webView(_: WKWebView, didFail _: WKNavigation?, withError error: any Error) { + self.controller?.errorText = error.localizedDescription + } +} + +private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler { + static let messageName = "openclawCanvasA2UIAction" + static let handlerNames = [messageName] + + weak var controller: ScreenController? + + func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) { + guard Self.handlerNames.contains(message.name) else { return } + guard let controller else { return } + + guard let url = message.webView?.url else { return } + if url.isFileURL { + guard controller.isTrustedCanvasUIURL(url) else { return } + } else { + // For security, only accept actions from local-network pages (e.g. the canvas host). + guard controller.isLocalNetworkCanvasURL(url) else { return } + } + + guard let body = ScreenController.parseA2UIActionBody(message.body) else { return } + + controller.onA2UIAction?(body) } } diff --git a/apps/ios/Tests/ScreenControllerTests.swift b/apps/ios/Tests/ScreenControllerTests.swift index 32c36acacb..d0e47c84fb 100644 --- a/apps/ios/Tests/ScreenControllerTests.swift +++ b/apps/ios/Tests/ScreenControllerTests.swift @@ -2,25 +2,38 @@ import Testing import WebKit @testable import OpenClaw +@MainActor +private func mountScreen(_ screen: ScreenController) throws -> (ScreenWebViewCoordinator, WKWebView) { + let coordinator = ScreenWebViewCoordinator(controller: screen) + _ = coordinator.makeContainerView() + let webView = try #require(coordinator.managedWebView) + return (coordinator, webView) +} + @Suite struct ScreenControllerTests { - @Test @MainActor func canvasModeConfiguresWebViewForTouch() { + @Test @MainActor func canvasModeConfiguresWebViewForTouch() throws { let screen = ScreenController() + let (coordinator, webView) = try mountScreen(screen) + defer { coordinator.teardown() } - #expect(screen.webView.isOpaque == true) - #expect(screen.webView.backgroundColor == .black) + #expect(webView.isOpaque == true) + #expect(webView.backgroundColor == .black) - let scrollView = screen.webView.scrollView + let scrollView = webView.scrollView #expect(scrollView.backgroundColor == .black) #expect(scrollView.contentInsetAdjustmentBehavior == .never) #expect(scrollView.isScrollEnabled == false) #expect(scrollView.bounces == false) } - @Test @MainActor func navigateEnablesScrollForWebPages() { + @Test @MainActor func navigateEnablesScrollForWebPages() throws { let screen = ScreenController() + let (coordinator, webView) = try mountScreen(screen) + defer { coordinator.teardown() } + screen.navigate(to: "https://example.com") - let scrollView = screen.webView.scrollView + let scrollView = webView.scrollView #expect(scrollView.isScrollEnabled == true) #expect(scrollView.bounces == true) } @@ -34,6 +47,9 @@ import WebKit @Test @MainActor func evalExecutesJavaScript() async throws { let screen = ScreenController() + let (coordinator, _) = try mountScreen(screen) + defer { coordinator.teardown() } + let deadline = ContinuousClock().now.advanced(by: .seconds(3)) while true {