fix(ios): refactor screen webview lifecycle handling (#20366)

Merged via /review-pr -> /prepare-pr -> /merge-pr.

Prepared head SHA: 7beb794a06
Co-authored-by: ngutman <1540134+ngutman@users.noreply.github.com>
Co-authored-by: ngutman <1540134+ngutman@users.noreply.github.com>
Reviewed-by: @ngutman
This commit is contained in:
Nimrod Gutman
2026-02-19 05:05:40 +08:00
committed by GitHub
parent e67da1538c
commit dd28a77df0
5 changed files with 276 additions and 145 deletions

View File

@@ -15,6 +15,7 @@ Docs: https://docs.openclaw.ai
### Fixes
- iOS/Screen: move `WKWebView` lifecycle ownership into `ScreenWebView` coordinator and explicit attach/detach flow to reduce gesture/lifecycle crash risk (`__NSArrayM insertObject:atIndex:` paths) during screen tab updates. (#20366) Thanks @ngutman.
- Protocol/Apple: regenerate Swift gateway models for `push.test` so `pnpm protocol:check` stays green on main. Thanks @mbelinky.
- Canvas/A2UI: improve bundled-asset resolution and empty-state handling so UI fallbacks render reliably. (#20312) Thanks @mbelinky.
- UI/Sessions: accept the canonical main session-key alias in Chat UI flows so main-session routing stays consistent. (#20311) Thanks @mbelinky.

View File

@@ -73,26 +73,6 @@ final class NodeAppModel {
var lastShareEventText: String = "No share events yet."
var openChatRequestID: Int = 0
var mainSessionKey: String {
let base = SessionKey.normalizeMainKey(self.mainSessionBaseKey)
let agentId = (self.selectedAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let defaultId = (self.gatewayDefaultAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
if agentId.isEmpty || (!defaultId.isEmpty && agentId == defaultId) { return base }
return SessionKey.makeAgentSessionKey(agentId: agentId, baseKey: base)
}
var activeAgentName: String {
let agentId = (self.selectedAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let defaultId = (self.gatewayDefaultAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let resolvedId = agentId.isEmpty ? defaultId : agentId
if resolvedId.isEmpty { return "Main" }
if let match = self.gatewayAgents.first(where: { $0.id == resolvedId }) {
let name = (match.name ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
return name.isEmpty ? match.id : name
}
return resolvedId
}
// Primary "node" connection: used for device capabilities and node.invoke requests.
private let nodeGateway = GatewayNodeSession()
// Secondary "operator" connection: used for chat/talk/config/voicewake requests.
@@ -1615,6 +1595,26 @@ private extension NodeAppModel {
}
extension NodeAppModel {
var mainSessionKey: String {
let base = SessionKey.normalizeMainKey(self.mainSessionBaseKey)
let agentId = (self.selectedAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let defaultId = (self.gatewayDefaultAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
if agentId.isEmpty || (!defaultId.isEmpty && agentId == defaultId) { return base }
return SessionKey.makeAgentSessionKey(agentId: agentId, baseKey: base)
}
var activeAgentName: String {
let agentId = (self.selectedAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let defaultId = (self.gatewayDefaultAgentId ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
let resolvedId = agentId.isEmpty ? defaultId : agentId
if resolvedId.isEmpty { return "Main" }
if let match = self.gatewayAgents.first(where: { $0.id == resolvedId }) {
let name = (match.name ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
return name.isEmpty ? match.id : name
}
return resolvedId
}
func connectToGateway(
url: URL,
gatewayStableID: String,

View File

@@ -1,14 +1,12 @@
import OpenClawKit
import Observation
import SwiftUI
import UIKit
import WebKit
@MainActor
@Observable
final class ScreenController {
let webView: WKWebView
private let navigationDelegate: ScreenNavigationDelegate
private let a2uiActionHandler: CanvasA2UIActionMessageHandler
private weak var activeWebView: WKWebView?
var urlString: String = ""
var errorText: String?
@@ -24,29 +22,6 @@ final class ScreenController {
private var debugStatusSubtitle: String?
init() {
let config = WKWebViewConfiguration()
config.websiteDataStore = .nonPersistent()
let a2uiActionHandler = CanvasA2UIActionMessageHandler()
let userContentController = WKUserContentController()
for name in CanvasA2UIActionMessageHandler.handlerNames {
userContentController.add(a2uiActionHandler, name: name)
}
config.userContentController = userContentController
self.navigationDelegate = ScreenNavigationDelegate()
self.a2uiActionHandler = a2uiActionHandler
self.webView = WKWebView(frame: .zero, configuration: config)
// Canvas scaffold is a fully self-contained HTML page; avoid relying on transparency underlays.
self.webView.isOpaque = true
self.webView.backgroundColor = .black
self.webView.scrollView.backgroundColor = .black
self.webView.scrollView.contentInsetAdjustmentBehavior = .never
self.webView.scrollView.contentInset = .zero
self.webView.scrollView.scrollIndicatorInsets = .zero
self.webView.scrollView.automaticallyAdjustsScrollIndicatorInsets = false
self.applyScrollBehavior()
self.webView.navigationDelegate = self.navigationDelegate
self.navigationDelegate.controller = self
a2uiActionHandler.controller = self
self.reload()
}
@@ -71,24 +46,26 @@ final class ScreenController {
}
func reload() {
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
self.applyScrollBehavior()
guard let webView = self.activeWebView else { return }
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty {
guard let url = Self.canvasScaffoldURL else { return }
self.errorText = nil
self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent())
webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent())
return
}
guard let url = URL(string: trimmed) else {
self.errorText = "Invalid URL: \(trimmed)"
return
}
self.errorText = nil
if url.isFileURL {
webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent())
} else {
guard let url = URL(string: trimmed) else {
self.errorText = "Invalid URL: \(trimmed)"
return
}
self.errorText = nil
if url.isFileURL {
self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent())
} else {
self.webView.load(URLRequest(url: url))
}
webView.load(URLRequest(url: url))
}
}
@@ -108,7 +85,8 @@ final class ScreenController {
self.applyDebugStatusIfNeeded()
}
fileprivate func applyDebugStatusIfNeeded() {
func applyDebugStatusIfNeeded() {
guard let webView = self.activeWebView else { return }
let enabled = self.debugStatusEnabled
let title = self.debugStatusTitle
let subtitle = self.debugStatusSubtitle
@@ -127,7 +105,7 @@ final class ScreenController {
} catch (_) {}
})()
"""
self.webView.evaluateJavaScript(js) { _, _ in }
webView.evaluateJavaScript(js) { _, _ in }
}
func waitForA2UIReady(timeoutMs: Int) async -> Bool {
@@ -154,8 +132,13 @@ final class ScreenController {
}
func eval(javaScript: String) async throws -> String {
try await withCheckedThrowingContinuation { cont in
self.webView.evaluateJavaScript(javaScript) { result, error in
guard let webView = self.activeWebView else {
throw NSError(domain: "Screen", code: 3, userInfo: [
NSLocalizedDescriptionKey: "web view unavailable",
])
}
return try await withCheckedThrowingContinuation { cont in
webView.evaluateJavaScript(javaScript) { result, error in
if let error {
cont.resume(throwing: error)
return
@@ -174,8 +157,13 @@ final class ScreenController {
if let maxWidth {
config.snapshotWidth = NSNumber(value: Double(maxWidth))
}
guard let webView = self.activeWebView else {
throw NSError(domain: "Screen", code: 3, userInfo: [
NSLocalizedDescriptionKey: "web view unavailable",
])
}
let image: UIImage = try await withCheckedThrowingContinuation { cont in
self.webView.takeSnapshot(with: config) { image, error in
webView.takeSnapshot(with: config) { image, error in
if let error {
cont.resume(throwing: error)
return
@@ -206,8 +194,13 @@ final class ScreenController {
if let maxWidth {
config.snapshotWidth = NSNumber(value: Double(maxWidth))
}
guard let webView = self.activeWebView else {
throw NSError(domain: "Screen", code: 3, userInfo: [
NSLocalizedDescriptionKey: "web view unavailable",
])
}
let image: UIImage = try await withCheckedThrowingContinuation { cont in
self.webView.takeSnapshot(with: config) { image, error in
webView.takeSnapshot(with: config) { image, error in
if let error {
cont.resume(throwing: error)
return
@@ -238,6 +231,17 @@ final class ScreenController {
return data.base64EncodedString()
}
func attachWebView(_ webView: WKWebView) {
self.activeWebView = webView
self.reload()
self.applyDebugStatusIfNeeded()
}
func detachWebView(_ webView: WKWebView) {
guard self.activeWebView === webView else { return }
self.activeWebView = nil
}
private static func bundledResourceURL(
name: String,
ext: String,
@@ -277,9 +281,10 @@ final class ScreenController {
}
private func applyScrollBehavior() {
guard let webView = self.activeWebView else { return }
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
let allowScroll = !trimmed.isEmpty
let scrollView = self.webView.scrollView
let scrollView = webView.scrollView
// Default canvas needs raw touch events; external pages should scroll.
scrollView.isScrollEnabled = allowScroll
scrollView.bounces = allowScroll
@@ -366,72 +371,3 @@ extension Double {
return self
}
}
// MARK: - Navigation Delegate
/// Handles navigation policy to intercept openclaw:// deep links from canvas
@MainActor
private final class ScreenNavigationDelegate: NSObject, WKNavigationDelegate {
weak var controller: ScreenController?
func webView(
_ webView: WKWebView,
decidePolicyFor navigationAction: WKNavigationAction,
decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void)
{
guard let url = navigationAction.request.url else {
decisionHandler(.allow)
return
}
// Intercept openclaw:// deep links.
if url.scheme?.lowercased() == "openclaw" {
decisionHandler(.cancel)
self.controller?.onDeepLink?(url)
return
}
decisionHandler(.allow)
}
func webView(
_: WKWebView,
didFailProvisionalNavigation _: WKNavigation?,
withError error: any Error)
{
self.controller?.errorText = error.localizedDescription
}
func webView(_: WKWebView, didFinish _: WKNavigation?) {
self.controller?.errorText = nil
self.controller?.applyDebugStatusIfNeeded()
}
func webView(_: WKWebView, didFail _: WKNavigation?, withError error: any Error) {
self.controller?.errorText = error.localizedDescription
}
}
private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler {
static let messageName = "openclawCanvasA2UIAction"
static let handlerNames = [messageName]
weak var controller: ScreenController?
func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) {
guard Self.handlerNames.contains(message.name) else { return }
guard let controller else { return }
guard let url = message.webView?.url else { return }
if url.isFileURL {
guard controller.isTrustedCanvasUIURL(url) else { return }
} else {
// For security, only accept actions from local-network pages (e.g. the canvas host).
guard controller.isLocalNetworkCanvasURL(url) else { return }
}
guard let body = ScreenController.parseA2UIActionBody(message.body) else { return }
controller.onA2UIAction?(body)
}
}

View File

@@ -5,11 +5,189 @@ import WebKit
struct ScreenWebView: UIViewRepresentable {
var controller: ScreenController
func makeUIView(context: Context) -> WKWebView {
self.controller.webView
func makeCoordinator() -> ScreenWebViewCoordinator {
ScreenWebViewCoordinator(controller: self.controller)
}
func updateUIView(_ webView: WKWebView, context: Context) {
// State changes are driven by ScreenController.
func makeUIView(context: Context) -> UIView {
context.coordinator.makeContainerView()
}
func updateUIView(_: UIView, context: Context) {
context.coordinator.updateController(self.controller)
}
static func dismantleUIView(_: UIView, coordinator: ScreenWebViewCoordinator) {
coordinator.teardown()
}
}
@MainActor
final class ScreenWebViewCoordinator: NSObject {
private weak var controller: ScreenController?
private let navigationDelegate = ScreenNavigationDelegate()
private let a2uiActionHandler = CanvasA2UIActionMessageHandler()
private let userContentController = WKUserContentController()
private(set) var managedWebView: WKWebView?
private weak var containerView: UIView?
init(controller: ScreenController) {
self.controller = controller
super.init()
self.navigationDelegate.controller = controller
self.a2uiActionHandler.controller = controller
}
func makeContainerView() -> UIView {
if let containerView {
return containerView
}
let container = UIView(frame: .zero)
container.backgroundColor = .black
let webView = Self.makeWebView(userContentController: self.userContentController)
webView.navigationDelegate = self.navigationDelegate
self.installA2UIHandlers()
webView.translatesAutoresizingMaskIntoConstraints = false
container.addSubview(webView)
NSLayoutConstraint.activate([
webView.leadingAnchor.constraint(equalTo: container.leadingAnchor),
webView.trailingAnchor.constraint(equalTo: container.trailingAnchor),
webView.topAnchor.constraint(equalTo: container.topAnchor),
webView.bottomAnchor.constraint(equalTo: container.bottomAnchor),
])
self.managedWebView = webView
self.containerView = container
self.controller?.attachWebView(webView)
return container
}
func updateController(_ controller: ScreenController) {
let previousController = self.controller
let controllerChanged = self.controller !== controller
self.controller = controller
self.navigationDelegate.controller = controller
self.a2uiActionHandler.controller = controller
if controllerChanged, let managedWebView {
previousController?.detachWebView(managedWebView)
controller.attachWebView(managedWebView)
}
}
func teardown() {
if let managedWebView {
self.controller?.detachWebView(managedWebView)
managedWebView.navigationDelegate = nil
}
self.removeA2UIHandlers()
self.navigationDelegate.controller = nil
self.a2uiActionHandler.controller = nil
self.managedWebView = nil
self.containerView = nil
}
private static func makeWebView(userContentController: WKUserContentController) -> WKWebView {
let config = WKWebViewConfiguration()
config.websiteDataStore = .nonPersistent()
config.userContentController = userContentController
let webView = WKWebView(frame: .zero, configuration: config)
// Canvas scaffold is a fully self-contained HTML page; avoid relying on transparency underlays.
webView.isOpaque = true
webView.backgroundColor = .black
let scrollView = webView.scrollView
scrollView.backgroundColor = .black
scrollView.contentInsetAdjustmentBehavior = .never
scrollView.contentInset = .zero
scrollView.scrollIndicatorInsets = .zero
scrollView.automaticallyAdjustsScrollIndicatorInsets = false
return webView
}
private func installA2UIHandlers() {
for name in CanvasA2UIActionMessageHandler.handlerNames {
self.userContentController.add(self.a2uiActionHandler, name: name)
}
}
private func removeA2UIHandlers() {
for name in CanvasA2UIActionMessageHandler.handlerNames {
self.userContentController.removeScriptMessageHandler(forName: name)
}
}
}
// MARK: - Navigation Delegate
/// Handles navigation policy to intercept openclaw:// deep links from canvas
@MainActor
private final class ScreenNavigationDelegate: NSObject, WKNavigationDelegate {
weak var controller: ScreenController?
func webView(
_: WKWebView,
decidePolicyFor navigationAction: WKNavigationAction,
decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void)
{
guard let url = navigationAction.request.url else {
decisionHandler(.allow)
return
}
// Intercept openclaw:// deep links.
if url.scheme?.lowercased() == "openclaw" {
decisionHandler(.cancel)
self.controller?.onDeepLink?(url)
return
}
decisionHandler(.allow)
}
func webView(
_: WKWebView,
didFailProvisionalNavigation _: WKNavigation?,
withError error: any Error)
{
self.controller?.errorText = error.localizedDescription
}
func webView(_: WKWebView, didFinish _: WKNavigation?) {
self.controller?.errorText = nil
self.controller?.applyDebugStatusIfNeeded()
}
func webView(_: WKWebView, didFail _: WKNavigation?, withError error: any Error) {
self.controller?.errorText = error.localizedDescription
}
}
private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler {
static let messageName = "openclawCanvasA2UIAction"
static let handlerNames = [messageName]
weak var controller: ScreenController?
func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) {
guard Self.handlerNames.contains(message.name) else { return }
guard let controller else { return }
guard let url = message.webView?.url else { return }
if url.isFileURL {
guard controller.isTrustedCanvasUIURL(url) else { return }
} else {
// For security, only accept actions from local-network pages (e.g. the canvas host).
guard controller.isLocalNetworkCanvasURL(url) else { return }
}
guard let body = ScreenController.parseA2UIActionBody(message.body) else { return }
controller.onA2UIAction?(body)
}
}

View File

@@ -2,25 +2,38 @@ import Testing
import WebKit
@testable import OpenClaw
@MainActor
private func mountScreen(_ screen: ScreenController) throws -> (ScreenWebViewCoordinator, WKWebView) {
let coordinator = ScreenWebViewCoordinator(controller: screen)
_ = coordinator.makeContainerView()
let webView = try #require(coordinator.managedWebView)
return (coordinator, webView)
}
@Suite struct ScreenControllerTests {
@Test @MainActor func canvasModeConfiguresWebViewForTouch() {
@Test @MainActor func canvasModeConfiguresWebViewForTouch() throws {
let screen = ScreenController()
let (coordinator, webView) = try mountScreen(screen)
defer { coordinator.teardown() }
#expect(screen.webView.isOpaque == true)
#expect(screen.webView.backgroundColor == .black)
#expect(webView.isOpaque == true)
#expect(webView.backgroundColor == .black)
let scrollView = screen.webView.scrollView
let scrollView = webView.scrollView
#expect(scrollView.backgroundColor == .black)
#expect(scrollView.contentInsetAdjustmentBehavior == .never)
#expect(scrollView.isScrollEnabled == false)
#expect(scrollView.bounces == false)
}
@Test @MainActor func navigateEnablesScrollForWebPages() {
@Test @MainActor func navigateEnablesScrollForWebPages() throws {
let screen = ScreenController()
let (coordinator, webView) = try mountScreen(screen)
defer { coordinator.teardown() }
screen.navigate(to: "https://example.com")
let scrollView = screen.webView.scrollView
let scrollView = webView.scrollView
#expect(scrollView.isScrollEnabled == true)
#expect(scrollView.bounces == true)
}
@@ -34,6 +47,9 @@ import WebKit
@Test @MainActor func evalExecutesJavaScript() async throws {
let screen = ScreenController()
let (coordinator, _) = try mountScreen(screen)
defer { coordinator.teardown() }
let deadline = ContinuousClock().now.advanced(by: .seconds(3))
while true {