github/openclaw
1
1
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-02-19 18:39:20 -05:00
Code Issues Packages Projects Releases Wiki Activity

Canvas: fix A2UI v0.8 rendering

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-17 13:20:27 +01:00
parent 81a9439eb2
commit 9eaa45a291
14 changed files with 301 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/macos/Sources/Clawdis/CanvasManager.swift
View File

@@ -93,7 +93,7 @@ final class CanvasManager {
func eval(sessionKey: String, javaScript: String) async throws -> String {
_ = try self.show(sessionKey: sessionKey, path: nil)
guard let controller = self.panelController else { return "" }
return await controller.eval(javaScript: javaScript)
return try await controller.eval(javaScript: javaScript)
}
func snapshot(sessionKey: String, outPath: String?) async throws -> String {

6
apps/macos/Sources/Clawdis/CanvasWindow.swift
View File

@@ -171,11 +171,11 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
self.webView.loadFileURL(fileURL, allowingReadAccessTo: accessDir)
}
func eval(javaScript: String) async -> String {
await withCheckedContinuation { cont in
func eval(javaScript: String) async throws -> String {
try await withCheckedThrowingContinuation { cont in
self.webView.evaluateJavaScript(javaScript) { result, error in
if let error {
cont.resume(returning: "error: \(error.localizedDescription)")
cont.resume(throwing: error)
return
}
if let result {

90
apps/macos/Sources/Clawdis/ControlRequestHandler.swift
View File

@@ -247,9 +247,12 @@ enum ControlRequestHandler {
case .reset:
js = """
(() => {
if (!globalThis.clawdisA2UI) { return "missing clawdisA2UI"; }
globalThis.clawdisA2UI.reset();
return "ok";
try {
if (!globalThis.clawdisA2UI) { return JSON.stringify({ ok: false, error: "missing clawdisA2UI" }); }
return JSON.stringify(globalThis.clawdisA2UI.reset());
} catch (e) {
return JSON.stringify({ ok: false, error: String(e?.message ?? e), stack: e?.stack });
}
})()
"""
@@ -257,43 +260,100 @@ enum ControlRequestHandler {
guard let jsonl, !jsonl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
return Response(ok: false, message: "missing jsonl")
}
let messages: [Any]
let items: [ParsedJSONLItem]
do {
messages = try Self.parseJSONL(jsonl)
items = try Self.parseJSONL(jsonl)
} catch {
return Response(ok: false, message: "invalid jsonl: \(error.localizedDescription)")
}
do {
try Self.validateA2UIV0_8(items)
} catch {
return Response(ok: false, message: error.localizedDescription)
}
let messages = items.map(\.value)
let data = try JSONSerialization.data(withJSONObject: messages, options: [])
let json = String(data: data, encoding: .utf8) ?? "[]"
js = """
(() => {
if (!globalThis.clawdisA2UI) { return "missing clawdisA2UI"; }
const messages = \(json);
globalThis.clawdisA2UI.applyMessages(messages);
return "ok";
try {
if (!globalThis.clawdisA2UI) { return JSON.stringify({ ok: false, error: "missing clawdisA2UI" }); }
const messages = \(json);
return JSON.stringify(globalThis.clawdisA2UI.applyMessages(messages));
} catch (e) {
return JSON.stringify({ ok: false, error: String(e?.message ?? e), stack: e?.stack });
}
})()
"""
}
let result = try await CanvasManager.shared.eval(sessionKey: session, javaScript: js)
return Response(ok: true, payload: Data(result.utf8))
let payload = Data(result.utf8)
if let obj = try? JSONSerialization.jsonObject(with: payload, options: []) as? [String: Any],
let ok = obj["ok"] as? Bool
{
let error = obj["error"] as? String
return Response(ok: ok, message: ok ? "" : (error ?? "A2UI error"), payload: payload)
}
return Response(ok: true, payload: payload)
} catch {
return Response(ok: false, message: error.localizedDescription)
}
}
private static func parseJSONL(_ text: String) throws -> [Any] {
var out: [Any] = []
for rawLine in text.split(whereSeparator: \.isNewline) {
let line = rawLine.trimmingCharacters(in: .whitespacesAndNewlines)
private struct ParsedJSONLItem {
let lineNumber: Int
let value: Any
}
private static func parseJSONL(_ text: String) throws -> [ParsedJSONLItem] {
var out: [ParsedJSONLItem] = []
var lineNumber = 0
for rawLine in text.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline) {
lineNumber += 1
let line = String(rawLine).trimmingCharacters(in: .whitespacesAndNewlines)
if line.isEmpty { continue }
let data = Data(line.utf8)
let obj = try JSONSerialization.jsonObject(with: data, options: [])
out.append(obj)
out.append(ParsedJSONLItem(lineNumber: lineNumber, value: obj))
}
return out
}
private static func validateA2UIV0_8(_ items: [ParsedJSONLItem]) throws {
let allowed = Set(["beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface"])
for item in items {
guard let dict = item.value as? [String: Any] else {
throw NSError(domain: "A2UI", code: 1, userInfo: [
NSLocalizedDescriptionKey: "A2UI JSONL line \(item.lineNumber): expected a JSON object",
])
}
if dict.keys.contains("createSurface") {
throw NSError(domain: "A2UI", code: 2, userInfo: [
NSLocalizedDescriptionKey: """
A2UI JSONL line \(item.lineNumber): looks like A2UI v0.9 (`createSurface`).
Canvas currently supports A2UI v0.8 server→client messages (`beginRendering`, `surfaceUpdate`, `dataModelUpdate`, `deleteSurface`).
""",
])
}
let matched = dict.keys.filter { allowed.contains($0) }
if matched.count != 1 {
let found = dict.keys.sorted().joined(separator: ", ")
throw NSError(domain: "A2UI", code: 3, userInfo: [
NSLocalizedDescriptionKey: """
A2UI JSONL line \(item.lineNumber): expected exactly one of \(allowed.sorted().joined(separator: ", ")); found: \(found)
""",
])
}
}
}
private static func waitForCanvasA2UI(session: String, timeoutMs: Int) async -> Bool {
let clock = ContinuousClock()
let deadline = clock.now.advanced(by: .milliseconds(timeoutMs))

199
apps/macos/Sources/Clawdis/Resources/CanvasA2UI/a2ui.bundle.js
View File

File diff suppressed because one or more lines are too long

2
apps/macos/Sources/ClawdisCLI/ClawdisCLI.swift
View File

@@ -515,7 +515,7 @@ struct ClawdisCLI {
Canvas:
clawdis-mac canvas show [--session <key>] [--target </...|https://...|file://...>]
[--x <screenX> --y <screenY>] [--width <w> --height <h>]
clawdis-mac canvas a2ui push --jsonl <path> [--session <key>]
clawdis-mac canvas a2ui push --jsonl <path> [--session <key>] # A2UI v0.8 JSONL
clawdis-mac canvas a2ui reset [--session <key>]
clawdis-mac canvas hide [--session <key>]
clawdis-mac canvas eval --js <code> [--session <key>]

16
docs/mac/canvas.md
View File

@@ -103,13 +103,27 @@ Related:
### Canvas A2UI
Canvas includes a built-in A2UI renderer (Lit-based). The agent can drive it with JSONL “message” objects:
Canvas includes a built-in **A2UI v0.8** renderer (Lit-based). The agent can drive it with JSONL **server→client protocol messages** (one JSON object per line):
- `clawdis-mac canvas a2ui push --jsonl <path> [--session <key>]`
- `clawdis-mac canvas a2ui reset [--session <key>]`
`push` expects a JSONL file where **each line is a single JSON object** (parsed and forwarded to the in-page A2UI renderer).
Minimal example (v0.8):
```bash
cat > /tmp/a2ui-v0.8.jsonl <<'EOF'
{"surfaceUpdate":{"surfaceId":"main","components":[{"id":"root","component":{"Column":{"children":{"explicitList":["title","content"]}}}},{"id":"title","component":{"Text":{"text":{"literalString":"Canvas (A2UI v0.8)"},"usageHint":"h1"}}},{"id":"content","component":{"Text":{"text":{"literalString":"If you can read this, `canvas a2ui push` works."},"usageHint":"body"}}}]}}
{"beginRendering":{"surfaceId":"main","root":"root"}}
EOF
clawdis-mac canvas a2ui push --jsonl /tmp/a2ui-v0.8.jsonl --session main
```
Notes:
- This does **not** support the A2UI v0.9 examples using `createSurface`.
## Triggering agent runs from Canvas (deep links)
Canvas can trigger new agent runs via the macOS app deep-link scheme:

2
vendor/a2ui/renderers/lit/dist/.tsbuildinfo vendored
View File

File diff suppressed because one or more lines are too long

2
vendor/a2ui/renderers/lit/dist/src/0.8/data/model-processor.d.ts.map vendored
View File

@@ -1 +1 @@
{"version":3,"file":"model-processor.d.ts","sourceRoot":"","sources":["../../../../src/0.8/data/model-processor.ts"],"names":[],"mappings":"AAgBA,OAAO,EACL,qBAAqB,EACrB,gBAAgB,EAKhB,SAAS,EAIT,OAAO,EAGP,gBAAgB,EAGjB,MAAM,gBAAgB,CAAC;AA0BxB;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,gBAAgB;;IAUzD,QAAQ,CAAC,IAAI,EAAE;QACb,OAAO,EAAE,cAAc,CAAC;QACxB,SAAS,EAAE,gBAAgB,CAAC;QAC5B,OAAO,EAAE,cAAc,CAAC;QACxB,OAAO,EAAE,iBAAiB,CAAC;KAC5B;IAdH,MAAM,CAAC,QAAQ,CAAC,kBAAkB,cAAc;gBASrC,IAAI,GAAE;QACb,OAAO,EAAE,cAAc,CAAC;QACxB,SAAS,EAAE,gBAAgB,CAAC;QAC5B,OAAO,EAAE,cAAc,CAAC;QACxB,OAAO,EAAE,iBAAiB,CAAC;KACwC;IAUvE,WAAW,IAAI,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC;IAI3C,aAAa;IAIb,eAAe,CAAC,QAAQ,EAAE,qBAAqB,EAAE,GAAG,IAAI;IA6BxD;;;;OAIG;IACH,OAAO,CACL,IAAI,EAAE,gBAAgB,EACtB,YAAY,EAAE,MAAM,EACpB,SAAS,SAA0C,GAClD,SAAS,GAAG,IAAI;IAkBnB,OAAO,CACL,IAAI,EAAE,gBAAgB,GAAG,IAAI,EAC7B,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,SAAS,EAChB,SAAS,SAA0C,GAClD,IAAI;IAuBP,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM;CAkqB5D"}
{"version":3,"file":"model-processor.d.ts","sourceRoot":"","sources":["../../../../src/0.8/data/model-processor.ts"],"names":[],"mappings":"AAgBA,OAAO,EACL,qBAAqB,EACrB,gBAAgB,EAKhB,SAAS,EAIT,OAAO,EAGP,gBAAgB,EAGjB,MAAM,gBAAgB,CAAC;AA0BxB;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,gBAAgB;;IAUzD,QAAQ,CAAC,IAAI,EAAE;QACb,OAAO,EAAE,cAAc,CAAC;QACxB,SAAS,EAAE,gBAAgB,CAAC;QAC5B,OAAO,EAAE,cAAc,CAAC;QACxB,OAAO,EAAE,iBAAiB,CAAC;KAC5B;IAdH,MAAM,CAAC,QAAQ,CAAC,kBAAkB,cAAc;gBASrC,IAAI,GAAE;QACb,OAAO,EAAE,cAAc,CAAC;QACxB,SAAS,EAAE,gBAAgB,CAAC;QAC5B,OAAO,EAAE,cAAc,CAAC;QACxB,OAAO,EAAE,iBAAiB,CAAC;KACwC;IAUvE,WAAW,IAAI,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC;IAI3C,aAAa;IAIb,eAAe,CAAC,QAAQ,EAAE,qBAAqB,EAAE,GAAG,IAAI;IA6BxD;;;;OAIG;IACH,OAAO,CACL,IAAI,EAAE,gBAAgB,EACtB,YAAY,EAAE,MAAM,EACpB,SAAS,SAA0C,GAClD,SAAS,GAAG,IAAI;IAkBnB,OAAO,CACL,IAAI,EAAE,gBAAgB,GAAG,IAAI,EAC7B,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,SAAS,EAChB,SAAS,SAA0C,GAClD,IAAI;IAuBP,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM;CA8qB5D"}

14
vendor/a2ui/renderers/lit/dist/src/0.8/data/model-processor.js vendored
View File

@@ -360,7 +360,7 @@ export class A2uiMessageProcessor {
const resolvedProperties = new this.#objCtor();
if (isObject(unresolvedProperties)) {
for (const [key, value] of Object.entries(unresolvedProperties)) {
resolvedProperties[key] = this.#resolvePropertyValue(value, surface, visited, dataContextPath, idSuffix);
resolvedProperties[key] = this.#resolvePropertyValue(value, surface, visited, dataContextPath, idSuffix, key);
}
}
visited.delete(fullId);
@@ -549,9 +549,13 @@ export class A2uiMessageProcessor {
* a child node (a string that matches a component ID), an explicitList of
* children, or a template, these will be built out here.
*/
#resolvePropertyValue(value, surface, visited, dataContextPath, idSuffix = "") {
#resolvePropertyValue(value, surface, visited, dataContextPath, idSuffix = "", propertyKey = null) {
const isComponentIdReferenceKey = (key) => key === "child" || key.endsWith("Child");
// 1. If it's a string that matches a component ID, build that node.
if (typeof value === "string" && surface.components.has(value)) {
if (typeof value === "string" &&
propertyKey &&
isComponentIdReferenceKey(propertyKey) &&
surface.components.has(value)) {
return this.#buildNodeRecursive(value, surface, visited, dataContextPath, idSuffix);
}
// 2. If it's a ComponentArrayReference (e.g., a `children` property),
@@ -597,7 +601,7 @@ export class A2uiMessageProcessor {
}
// 3. If it's a plain array, resolve each of its items.
if (Array.isArray(value)) {
return value.map((item) => this.#resolvePropertyValue(item, surface, visited, dataContextPath, idSuffix));
return value.map((item) => this.#resolvePropertyValue(item, surface, visited, dataContextPath, idSuffix, propertyKey));
}
// 4. If it's a plain object, resolve each of its properties.
if (isObject(value)) {
@@ -617,7 +621,7 @@ export class A2uiMessageProcessor {
newObj[key] = propertyValue;
continue;
}
newObj[key] = this.#resolvePropertyValue(propertyValue, surface, visited, dataContextPath, idSuffix);
newObj[key] = this.#resolvePropertyValue(propertyValue, surface, visited, dataContextPath, idSuffix, key);
}
return newObj;
}

2
vendor/a2ui/renderers/lit/dist/src/0.8/data/model-processor.js.map vendored
View File

File diff suppressed because one or more lines are too long

37
vendor/a2ui/renderers/lit/dist/src/0.8/model.test.js vendored
View File

@@ -314,6 +314,43 @@ describe("A2uiMessageProcessor", () => {
assert.strictEqual(plainTree.properties.children[0].id, "child");
assert.strictEqual(plainTree.properties.children[0].type, "Text");
});
it("should not treat enum-like strings as child component IDs", () => {
processor.processMessages([
{
surfaceUpdate: {
surfaceId: "@default",
components: [
{
id: "root",
component: {
Column: { children: { explicitList: ["body"] } },
},
},
{
id: "body",
component: {
Text: {
text: { literalString: "Hello" },
usageHint: "body",
},
},
},
],
},
},
{
beginRendering: {
root: "root",
surfaceId: "@default",
},
},
]);
const tree = processor.getSurfaces().get("@default")?.componentTree;
const plainTree = toPlainObject(tree);
assert.strictEqual(plainTree.id, "root");
assert.strictEqual(plainTree.properties.children[0].id, "body");
assert.strictEqual(plainTree.properties.children[0].type, "Text");
});
it("should throw an error on circular dependencies", () => {
// First, load the components
processor.processMessages([

2
vendor/a2ui/renderers/lit/dist/src/0.8/model.test.js.map vendored
View File

File diff suppressed because one or more lines are too long

22
vendor/a2ui/renderers/lit/src/0.8/data/model-processor.ts vendored
View File

@@ -507,7 +507,8 @@ export class A2uiMessageProcessor implements MessageProcessor {
surface,
visited,
dataContextPath,
idSuffix
idSuffix,
key
);
}
}
@@ -725,10 +726,19 @@ export class A2uiMessageProcessor implements MessageProcessor {
surface: Surface,
visited: Set<string>,
dataContextPath: string,
idSuffix = ""
idSuffix = "",
propertyKey: string | null = null
): ResolvedValue {
const isComponentIdReferenceKey = (key: string) =>
key === "child" || key.endsWith("Child");
// 1. If it's a string that matches a component ID, build that node.
if (typeof value === "string" && surface.components.has(value)) {
if (
typeof value === "string" &&
propertyKey &&
isComponentIdReferenceKey(propertyKey) &&
surface.components.has(value)
) {
return this.#buildNodeRecursive(
value,
surface,
@@ -814,7 +824,8 @@ export class A2uiMessageProcessor implements MessageProcessor {
surface,
visited,
dataContextPath,
idSuffix
idSuffix,
propertyKey
)
);
}
@@ -843,7 +854,8 @@ export class A2uiMessageProcessor implements MessageProcessor {
surface,
visited,
dataContextPath,
idSuffix
idSuffix,
key
);
}
return newObj;

39
vendor/a2ui/renderers/lit/src/0.8/model.test.ts vendored
View File

@@ -375,6 +375,45 @@ describe("A2uiMessageProcessor", () => {
assert.strictEqual(plainTree.properties.children[0].type, "Text");
});
it("should not treat enum-like strings as child component IDs", () => {
processor.processMessages([
{
surfaceUpdate: {
surfaceId: "@default",
components: [
{
id: "root",
component: {
Column: { children: { explicitList: ["body"] } },
},
},
{
id: "body",
component: {
Text: {
text: { literalString: "Hello" },
usageHint: "body",
},
},
},
],
},
},
{
beginRendering: {
root: "root",
surfaceId: "@default",
},
},
]);
const tree = processor.getSurfaces().get("@default")?.componentTree;
const plainTree = toPlainObject(tree);
assert.strictEqual(plainTree.id, "root");
assert.strictEqual(plainTree.properties.children[0].id, "body");
assert.strictEqual(plainTree.properties.children[0].type, "Text");
});
it("should throw an error on circular dependencies", () => {
// First, load the components
processor.processMessages([