feat: Android companion app improvements & gateway URL camera payloads (#13541)

Merged via /review-pr -> /prepare-pr -> /merge-pr.

Prepared head SHA: 9c179c9c31
Co-authored-by: smartprogrammer93 <33181301+smartprogrammer93@users.noreply.github.com>
Co-authored-by: steipete <58493+steipete@users.noreply.github.com>
Reviewed-by: @steipete
This commit is contained in:
Ahmad Bitar
2026-02-13 23:49:28 +08:00
committed by GitHub
parent 41f2f359a5
commit c179f71f42
38 changed files with 2158 additions and 748 deletions

View File

@@ -17,6 +17,7 @@ Docs: https://docs.openclaw.ai
- Security/Audit: distinguish external webhooks (`hooks.enabled`) from internal hooks (`hooks.internal.enabled`) in attack-surface summaries to avoid false exposure signals when only internal hooks are enabled. (#13474) Thanks @mcaxtr.
- Security/Onboarding: clarify multi-user DM isolation remediation with explicit `openclaw config set session.dmScope ...` commands in security audit, doctor security, and channel onboarding guidance. (#13129) Thanks @VintLin.
- Security/Audit: add misconfiguration checks for sandbox Docker config with sandbox mode off, ineffective `gateway.nodes.denyCommands` entries, global minimal tool-profile overrides by agent profiles, and permissive extension-plugin tool reachability.
- Android/Nodes: harden `app.update` by requiring HTTPS and gateway-host URL matching plus SHA-256 verification, stream URL camera downloads to disk with size guards to avoid memory spikes, and stop signing release builds with debug keys. (#13541) Thanks @smartprogrammer93.
- Auto-reply/Threading: auto-inject implicit reply threading so `replyToMode` works without requiring model-emitted `[[reply_to_current]]`, while preserving `replyToMode: "off"` behavior for implicit Slack replies and keeping block-streaming chunk coalescing stable under `replyToMode: "first"`. (#14976) Thanks @Diaspar4u.
- Sandbox: pass configured `sandbox.docker.env` variables to sandbox containers at `docker create` time. (#15138) Thanks @stevebot-alive.
- Onboarding/CLI: restore terminal state without resuming paused `stdin`, so onboarding exits cleanly after choosing Web UI and the installer returns instead of appearing stuck.

View File

@@ -23,10 +23,19 @@ android {
targetSdk = 36
versionCode = 202602130
versionName = "2026.2.13"
ndk {
// Support all major ABIs — native libs are tiny (~47 KB per ABI)
abiFilters += listOf("armeabi-v7a", "arm64-v8a", "x86", "x86_64")
}
}
buildTypes {
release {
isMinifyEnabled = true
isShrinkResources = true
proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro")
}
debug {
isMinifyEnabled = false
}
}
@@ -43,7 +52,13 @@ android {
packaging {
resources {
excludes += "/META-INF/{AL2.0,LGPL2.1}"
excludes += setOf(
"/META-INF/{AL2.0,LGPL2.1}",
"/META-INF/*.version",
"/META-INF/LICENSE*.txt",
"DebugProbesKt.bin",
"kotlin-tooling-metadata.json",
)
}
}
@@ -90,6 +105,8 @@ dependencies {
implementation("androidx.compose.ui:ui")
implementation("androidx.compose.ui:ui-tooling-preview")
implementation("androidx.compose.material3:material3")
// material-icons-extended pulled in full icon set (~20 MB DEX). Only ~18 icons used.
// R8 will tree-shake unused icons when minify is enabled on release builds.
implementation("androidx.compose.material:material-icons-extended")
implementation("androidx.navigation:navigation-compose:2.9.6")

28
apps/android/app/proguard-rules.pro vendored Normal file
View File

@@ -0,0 +1,28 @@
# ── App classes ───────────────────────────────────────────────────
-keep class ai.openclaw.android.** { *; }
# ── Bouncy Castle ─────────────────────────────────────────────────
-keep class org.bouncycastle.** { *; }
-dontwarn org.bouncycastle.**
# ── CameraX ───────────────────────────────────────────────────────
-keep class androidx.camera.** { *; }
# ── kotlinx.serialization ────────────────────────────────────────
-keep class kotlinx.serialization.** { *; }
-keepclassmembers class * {
@kotlinx.serialization.Serializable *;
}
-keepattributes *Annotation*, InnerClasses
# ── OkHttp ────────────────────────────────────────────────────────
-dontwarn okhttp3.**
-dontwarn okio.**
-keep class okhttp3.internal.platform.** { *; }
# ── Misc suppressions ────────────────────────────────────────────
-dontwarn com.sun.jna.**
-dontwarn javax.naming.**
-dontwarn lombok.Generated
-dontwarn org.slf4j.impl.StaticLoggerBinder
-dontwarn sun.net.spi.nameservice.NameServiceDescriptor

View File

@@ -15,6 +15,7 @@
<uses-permission android:name="android.permission.CAMERA" />
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="android.permission.SEND_SMS" />
<uses-permission android:name="android.permission.REQUEST_INSTALL_PACKAGES" />
<uses-feature
android:name="android.hardware.camera"
android:required="false" />
@@ -37,13 +38,27 @@
android:name=".NodeForegroundService"
android:exported="false"
android:foregroundServiceType="dataSync|microphone|mediaProjection" />
<provider
android:name="androidx.core.content.FileProvider"
android:authorities="${applicationId}.fileprovider"
android:exported="false"
android:grantUriPermissions="true">
<meta-data
android:name="android.support.FILE_PROVIDER_PATHS"
android:resource="@xml/file_paths" />
</provider>
<activity
android:name=".MainActivity"
android:exported="true">
android:exported="true"
android:configChanges="orientation|screenSize|screenLayout|smallestScreenSize|uiMode|density|keyboard|keyboardHidden|navigation">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
<receiver
android:name=".InstallResultReceiver"
android:exported="false" />
</application>
</manifest>

View File

@@ -0,0 +1,33 @@
package ai.openclaw.android
import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import android.content.pm.PackageInstaller
import android.util.Log
class InstallResultReceiver : BroadcastReceiver() {
override fun onReceive(context: Context, intent: Intent) {
val status = intent.getIntExtra(PackageInstaller.EXTRA_STATUS, PackageInstaller.STATUS_FAILURE)
val message = intent.getStringExtra(PackageInstaller.EXTRA_STATUS_MESSAGE)
when (status) {
PackageInstaller.STATUS_PENDING_USER_ACTION -> {
// System needs user confirmation — launch the confirmation activity
@Suppress("DEPRECATION")
val confirmIntent = intent.getParcelableExtra<Intent>(Intent.EXTRA_INTENT)
if (confirmIntent != null) {
confirmIntent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
context.startActivity(confirmIntent)
Log.w("openclaw", "app.update: user confirmation requested, launching install dialog")
}
}
PackageInstaller.STATUS_SUCCESS -> {
Log.w("openclaw", "app.update: install SUCCESS")
}
else -> {
Log.e("openclaw", "app.update: install FAILED status=$status message=$message")
}
}
}
}

View File

@@ -51,6 +51,7 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
val manualHost: StateFlow<String> = runtime.manualHost
val manualPort: StateFlow<Int> = runtime.manualPort
val manualTls: StateFlow<Boolean> = runtime.manualTls
val gatewayToken: StateFlow<String> = runtime.gatewayToken
val canvasDebugStatusEnabled: StateFlow<Boolean> = runtime.canvasDebugStatusEnabled
val chatSessionKey: StateFlow<String> = runtime.chatSessionKey
@@ -104,6 +105,10 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
runtime.setManualTls(value)
}
fun setGatewayToken(value: String) {
runtime.setGatewayToken(value)
}
fun setCanvasDebugStatusEnabled(value: Boolean) {
runtime.setCanvasDebugStatusEnabled(value)
}

View File

@@ -2,12 +2,23 @@ package ai.openclaw.android
import android.app.Application
import android.os.StrictMode
import android.util.Log
import java.security.Security
class NodeApp : Application() {
val runtime: NodeRuntime by lazy { NodeRuntime(this) }
override fun onCreate() {
super.onCreate()
// Register Bouncy Castle as highest-priority provider for Ed25519 support
try {
val bcProvider = Class.forName("org.bouncycastle.jce.provider.BouncyCastleProvider")
.getDeclaredConstructor().newInstance() as java.security.Provider
Security.removeProvider("BC")
Security.insertProviderAt(bcProvider, 1)
} catch (it: Throwable) {
Log.e("NodeApp", "Failed to register Bouncy Castle provider", it)
}
if (BuildConfig.DEBUG) {
StrictMode.setThreadPolicy(
StrictMode.ThreadPolicy.Builder()

View File

@@ -3,8 +3,6 @@ package ai.openclaw.android
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.location.LocationManager
import android.os.Build
import android.os.SystemClock
import androidx.core.content.ContextCompat
import ai.openclaw.android.chat.ChatController
@@ -14,45 +12,26 @@ import ai.openclaw.android.chat.ChatSessionEntry
import ai.openclaw.android.chat.OutgoingAttachment
import ai.openclaw.android.gateway.DeviceAuthStore
import ai.openclaw.android.gateway.DeviceIdentityStore
import ai.openclaw.android.gateway.GatewayClientInfo
import ai.openclaw.android.gateway.GatewayConnectOptions
import ai.openclaw.android.gateway.GatewayDiscovery
import ai.openclaw.android.gateway.GatewayEndpoint
import ai.openclaw.android.gateway.GatewaySession
import ai.openclaw.android.gateway.GatewayTlsParams
import ai.openclaw.android.node.CameraCaptureManager
import ai.openclaw.android.node.LocationCaptureManager
import ai.openclaw.android.BuildConfig
import ai.openclaw.android.node.CanvasController
import ai.openclaw.android.node.ScreenRecordManager
import ai.openclaw.android.node.SmsManager
import ai.openclaw.android.protocol.OpenClawCapability
import ai.openclaw.android.protocol.OpenClawCameraCommand
import ai.openclaw.android.node.*
import ai.openclaw.android.protocol.OpenClawCanvasA2UIAction
import ai.openclaw.android.protocol.OpenClawCanvasA2UICommand
import ai.openclaw.android.protocol.OpenClawCanvasCommand
import ai.openclaw.android.protocol.OpenClawScreenCommand
import ai.openclaw.android.protocol.OpenClawLocationCommand
import ai.openclaw.android.protocol.OpenClawSmsCommand
import ai.openclaw.android.voice.TalkModeManager
import ai.openclaw.android.voice.VoiceWakeManager
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.TimeoutCancellationException
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.flow.collect
import kotlinx.coroutines.flow.distinctUntilChanged
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
@@ -112,6 +91,80 @@ class NodeRuntime(context: Context) {
val discoveryStatusText: StateFlow<String> = discovery.statusText
private val identityStore = DeviceIdentityStore(appContext)
private var connectedEndpoint: GatewayEndpoint? = null
private val cameraHandler: CameraHandler = CameraHandler(
appContext = appContext,
camera = camera,
prefs = prefs,
connectedEndpoint = { connectedEndpoint },
externalAudioCaptureActive = externalAudioCaptureActive,
showCameraHud = ::showCameraHud,
triggerCameraFlash = ::triggerCameraFlash,
invokeErrorFromThrowable = { invokeErrorFromThrowable(it) },
)
private val debugHandler: DebugHandler = DebugHandler(
appContext = appContext,
identityStore = identityStore,
)
private val appUpdateHandler: AppUpdateHandler = AppUpdateHandler(
appContext = appContext,
connectedEndpoint = { connectedEndpoint },
)
private val locationHandler: LocationHandler = LocationHandler(
appContext = appContext,
location = location,
json = json,
isForeground = { _isForeground.value },
locationMode = { locationMode.value },
locationPreciseEnabled = { locationPreciseEnabled.value },
)
private val screenHandler: ScreenHandler = ScreenHandler(
screenRecorder = screenRecorder,
setScreenRecordActive = { _screenRecordActive.value = it },
invokeErrorFromThrowable = { invokeErrorFromThrowable(it) },
)
private val smsHandlerImpl: SmsHandler = SmsHandler(
sms = sms,
)
private val a2uiHandler: A2UIHandler = A2UIHandler(
canvas = canvas,
json = json,
getNodeCanvasHostUrl = { nodeSession.currentCanvasHostUrl() },
getOperatorCanvasHostUrl = { operatorSession.currentCanvasHostUrl() },
)
private val connectionManager: ConnectionManager = ConnectionManager(
prefs = prefs,
cameraEnabled = { cameraEnabled.value },
locationMode = { locationMode.value },
voiceWakeMode = { voiceWakeMode.value },
smsAvailable = { sms.canSendSms() },
hasRecordAudioPermission = { hasRecordAudioPermission() },
manualTls = { manualTls.value },
)
private val invokeDispatcher: InvokeDispatcher = InvokeDispatcher(
canvas = canvas,
cameraHandler = cameraHandler,
locationHandler = locationHandler,
screenHandler = screenHandler,
smsHandler = smsHandlerImpl,
a2uiHandler = a2uiHandler,
debugHandler = debugHandler,
appUpdateHandler = appUpdateHandler,
isForeground = { _isForeground.value },
cameraEnabled = { cameraEnabled.value },
locationEnabled = { locationMode.value != LocationMode.Off },
)
private lateinit var gatewayEventHandler: GatewayEventHandler
private val _isConnected = MutableStateFlow(false)
val isConnected: StateFlow<Boolean> = _isConnected.asStateFlow()
@@ -149,7 +202,6 @@ class NodeRuntime(context: Context) {
private var nodeConnected = false
private var operatorStatusText: String = "Offline"
private var nodeStatusText: String = "Offline"
private var connectedEndpoint: GatewayEndpoint? = null
private val operatorSession =
GatewaySession(
@@ -165,7 +217,7 @@ class NodeRuntime(context: Context) {
applyMainSessionKey(mainSessionKey)
updateStatus()
scope.launch { refreshBrandingFromGateway() }
scope.launch { refreshWakeWordsFromGateway() }
scope.launch { gatewayEventHandler.refreshWakeWordsFromGateway() }
},
onDisconnected = { message ->
operatorConnected = false
@@ -206,7 +258,7 @@ class NodeRuntime(context: Context) {
},
onEvent = { _, _ -> },
onInvoke = { req ->
handleInvoke(req.command, req.paramsJson)
invokeDispatcher.handleInvoke(req.command, req.paramsJson)
},
onTlsFingerprint = { stableId, fingerprint ->
prefs.saveGatewayTlsFingerprint(stableId, fingerprint)
@@ -231,8 +283,7 @@ class NodeRuntime(context: Context) {
}
private fun applyMainSessionKey(candidate: String?) {
val trimmed = candidate?.trim().orEmpty()
if (trimmed.isEmpty()) return
val trimmed = normalizeMainKey(candidate) ?: return
if (isCanonicalMainSessionKey(_mainSessionKey.value)) return
if (_mainSessionKey.value == trimmed) return
_mainSessionKey.value = trimmed
@@ -258,7 +309,7 @@ class NodeRuntime(context: Context) {
}
private fun maybeNavigateToA2uiOnConnect() {
val a2uiUrl = resolveA2uiHostUrl() ?: return
val a2uiUrl = a2uiHandler.resolveA2uiHostUrl() ?: return
val current = canvas.currentUrl()?.trim().orEmpty()
if (current.isEmpty() || current == lastAutoA2uiUrl) {
lastAutoA2uiUrl = a2uiUrl
@@ -284,12 +335,12 @@ class NodeRuntime(context: Context) {
val manualHost: StateFlow<String> = prefs.manualHost
val manualPort: StateFlow<Int> = prefs.manualPort
val manualTls: StateFlow<Boolean> = prefs.manualTls
val gatewayToken: StateFlow<String> = prefs.gatewayToken
fun setGatewayToken(value: String) = prefs.setGatewayToken(value)
val lastDiscoveredStableId: StateFlow<String> = prefs.lastDiscoveredStableId
val canvasDebugStatusEnabled: StateFlow<Boolean> = prefs.canvasDebugStatusEnabled
private var didAutoConnect = false
private var suppressWakeWordsSync = false
private var wakeWordsSyncJob: Job? = null
val chatSessionKey: StateFlow<String> = chat.sessionKey
val chatSessionId: StateFlow<String?> = chat.sessionId
@@ -303,6 +354,14 @@ class NodeRuntime(context: Context) {
val pendingRunCount: StateFlow<Int> = chat.pendingRunCount
init {
gatewayEventHandler = GatewayEventHandler(
scope = scope,
prefs = prefs,
json = json,
operatorSession = operatorSession,
isConnected = { _isConnected.value },
)
scope.launch {
combine(
voiceWakeMode,
@@ -434,7 +493,7 @@ class NodeRuntime(context: Context) {
fun setWakeWords(words: List<String>) {
prefs.setWakeWords(words)
scheduleWakeWordsSyncIfNeeded()
gatewayEventHandler.scheduleWakeWordsSyncIfNeeded()
}
fun resetWakeWordsDefaults() {
@@ -449,110 +508,13 @@ class NodeRuntime(context: Context) {
prefs.setTalkEnabled(value)
}
private fun buildInvokeCommands(): List<String> =
buildList {
add(OpenClawCanvasCommand.Present.rawValue)
add(OpenClawCanvasCommand.Hide.rawValue)
add(OpenClawCanvasCommand.Navigate.rawValue)
add(OpenClawCanvasCommand.Eval.rawValue)
add(OpenClawCanvasCommand.Snapshot.rawValue)
add(OpenClawCanvasA2UICommand.Push.rawValue)
add(OpenClawCanvasA2UICommand.PushJSONL.rawValue)
add(OpenClawCanvasA2UICommand.Reset.rawValue)
add(OpenClawScreenCommand.Record.rawValue)
if (cameraEnabled.value) {
add(OpenClawCameraCommand.Snap.rawValue)
add(OpenClawCameraCommand.Clip.rawValue)
}
if (locationMode.value != LocationMode.Off) {
add(OpenClawLocationCommand.Get.rawValue)
}
if (sms.canSendSms()) {
add(OpenClawSmsCommand.Send.rawValue)
}
}
private fun buildCapabilities(): List<String> =
buildList {
add(OpenClawCapability.Canvas.rawValue)
add(OpenClawCapability.Screen.rawValue)
if (cameraEnabled.value) add(OpenClawCapability.Camera.rawValue)
if (sms.canSendSms()) add(OpenClawCapability.Sms.rawValue)
if (voiceWakeMode.value != VoiceWakeMode.Off && hasRecordAudioPermission()) {
add(OpenClawCapability.VoiceWake.rawValue)
}
if (locationMode.value != LocationMode.Off) {
add(OpenClawCapability.Location.rawValue)
}
}
private fun resolvedVersionName(): String {
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
return if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
"$versionName-dev"
} else {
versionName
}
}
private fun resolveModelIdentifier(): String? {
return listOfNotNull(Build.MANUFACTURER, Build.MODEL)
.joinToString(" ")
.trim()
.ifEmpty { null }
}
private fun buildUserAgent(): String {
val version = resolvedVersionName()
val release = Build.VERSION.RELEASE?.trim().orEmpty()
val releaseLabel = if (release.isEmpty()) "unknown" else release
return "OpenClawAndroid/$version (Android $releaseLabel; SDK ${Build.VERSION.SDK_INT})"
}
private fun buildClientInfo(clientId: String, clientMode: String): GatewayClientInfo {
return GatewayClientInfo(
id = clientId,
displayName = displayName.value,
version = resolvedVersionName(),
platform = "android",
mode = clientMode,
instanceId = instanceId.value,
deviceFamily = "Android",
modelIdentifier = resolveModelIdentifier(),
)
}
private fun buildNodeConnectOptions(): GatewayConnectOptions {
return GatewayConnectOptions(
role = "node",
scopes = emptyList(),
caps = buildCapabilities(),
commands = buildInvokeCommands(),
permissions = emptyMap(),
client = buildClientInfo(clientId = "openclaw-android", clientMode = "node"),
userAgent = buildUserAgent(),
)
}
private fun buildOperatorConnectOptions(): GatewayConnectOptions {
return GatewayConnectOptions(
role = "operator",
scopes = emptyList(),
caps = emptyList(),
commands = emptyList(),
permissions = emptyMap(),
client = buildClientInfo(clientId = "openclaw-control-ui", clientMode = "ui"),
userAgent = buildUserAgent(),
)
}
fun refreshGatewayConnection() {
val endpoint = connectedEndpoint ?: return
val token = prefs.loadGatewayToken()
val password = prefs.loadGatewayPassword()
val tls = resolveTlsParams(endpoint)
operatorSession.connect(endpoint, token, password, buildOperatorConnectOptions(), tls)
nodeSession.connect(endpoint, token, password, buildNodeConnectOptions(), tls)
val tls = connectionManager.resolveTlsParams(endpoint)
operatorSession.connect(endpoint, token, password, connectionManager.buildOperatorConnectOptions(), tls)
nodeSession.connect(endpoint, token, password, connectionManager.buildNodeConnectOptions(), tls)
operatorSession.reconnect()
nodeSession.reconnect()
}
@@ -564,9 +526,9 @@ class NodeRuntime(context: Context) {
updateStatus()
val token = prefs.loadGatewayToken()
val password = prefs.loadGatewayPassword()
val tls = resolveTlsParams(endpoint)
operatorSession.connect(endpoint, token, password, buildOperatorConnectOptions(), tls)
nodeSession.connect(endpoint, token, password, buildNodeConnectOptions(), tls)
val tls = connectionManager.resolveTlsParams(endpoint)
operatorSession.connect(endpoint, token, password, connectionManager.buildOperatorConnectOptions(), tls)
nodeSession.connect(endpoint, token, password, connectionManager.buildNodeConnectOptions(), tls)
}
private fun hasRecordAudioPermission(): Boolean {
@@ -576,27 +538,6 @@ class NodeRuntime(context: Context) {
)
}
private fun hasFineLocationPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_FINE_LOCATION) ==
PackageManager.PERMISSION_GRANTED
)
}
private fun hasCoarseLocationPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_COARSE_LOCATION) ==
PackageManager.PERMISSION_GRANTED
)
}
private fun hasBackgroundLocationPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_BACKGROUND_LOCATION) ==
PackageManager.PERMISSION_GRANTED
)
}
fun connectManual() {
val host = manualHost.value.trim()
val port = manualPort.value
@@ -613,42 +554,6 @@ class NodeRuntime(context: Context) {
nodeSession.disconnect()
}
private fun resolveTlsParams(endpoint: GatewayEndpoint): GatewayTlsParams? {
val stored = prefs.loadGatewayTlsFingerprint(endpoint.stableId)
val hinted = endpoint.tlsEnabled || !endpoint.tlsFingerprintSha256.isNullOrBlank()
val manual = endpoint.stableId.startsWith("manual|")
if (manual) {
if (!manualTls.value) return null
return GatewayTlsParams(
required = true,
expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored,
allowTOFU = stored == null,
stableId = endpoint.stableId,
)
}
if (hinted) {
return GatewayTlsParams(
required = true,
expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored,
allowTOFU = stored == null,
stableId = endpoint.stableId,
)
}
if (!stored.isNullOrBlank()) {
return GatewayTlsParams(
required = true,
expectedFingerprint = stored,
allowTOFU = false,
stableId = endpoint.stableId,
)
}
return null
}
fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
scope.launch {
val trimmed = payloadJson.trim()
@@ -752,15 +657,7 @@ class NodeRuntime(context: Context) {
private fun handleGatewayEvent(event: String, payloadJson: String?) {
if (event == "voicewake.changed") {
if (payloadJson.isNullOrBlank()) return
try {
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
val array = payload["triggers"] as? JsonArray ?: return
val triggers = array.mapNotNull { it.asStringOrNull() }
applyWakeWordsFromGateway(triggers)
} catch (_: Throwable) {
// ignore
}
gatewayEventHandler.handleVoiceWakeChangedEvent(payloadJson)
return
}
@@ -768,44 +665,6 @@ class NodeRuntime(context: Context) {
chat.handleGatewayEvent(event, payloadJson)
}
private fun applyWakeWordsFromGateway(words: List<String>) {
suppressWakeWordsSync = true
prefs.setWakeWords(words)
suppressWakeWordsSync = false
}
private fun scheduleWakeWordsSyncIfNeeded() {
if (suppressWakeWordsSync) return
if (!_isConnected.value) return
val snapshot = prefs.wakeWords.value
wakeWordsSyncJob?.cancel()
wakeWordsSyncJob =
scope.launch {
delay(650)
val jsonList = snapshot.joinToString(separator = ",") { it.toJsonString() }
val params = """{"triggers":[$jsonList]}"""
try {
operatorSession.request("voicewake.set", params)
} catch (_: Throwable) {
// ignore
}
}
}
private suspend fun refreshWakeWordsFromGateway() {
if (!_isConnected.value) return
try {
val res = operatorSession.request("voicewake.get", "{}")
val payload = json.parseToJsonElement(res).asObjectOrNull() ?: return
val array = payload["triggers"] as? JsonArray ?: return
val triggers = array.mapNotNull { it.asStringOrNull() }
applyWakeWordsFromGateway(triggers)
} catch (_: Throwable) {
// ignore
}
}
private suspend fun refreshBrandingFromGateway() {
if (!_isConnected.value) return
try {
@@ -825,242 +684,6 @@ class NodeRuntime(context: Context) {
}
}
private suspend fun handleInvoke(command: String, paramsJson: String?): GatewaySession.InvokeResult {
if (
command.startsWith(OpenClawCanvasCommand.NamespacePrefix) ||
command.startsWith(OpenClawCanvasA2UICommand.NamespacePrefix) ||
command.startsWith(OpenClawCameraCommand.NamespacePrefix) ||
command.startsWith(OpenClawScreenCommand.NamespacePrefix)
) {
if (!isForeground.value) {
return GatewaySession.InvokeResult.error(
code = "NODE_BACKGROUND_UNAVAILABLE",
message = "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground",
)
}
}
if (command.startsWith(OpenClawCameraCommand.NamespacePrefix) && !cameraEnabled.value) {
return GatewaySession.InvokeResult.error(
code = "CAMERA_DISABLED",
message = "CAMERA_DISABLED: enable Camera in Settings",
)
}
if (command.startsWith(OpenClawLocationCommand.NamespacePrefix) &&
locationMode.value == LocationMode.Off
) {
return GatewaySession.InvokeResult.error(
code = "LOCATION_DISABLED",
message = "LOCATION_DISABLED: enable Location in Settings",
)
}
return when (command) {
OpenClawCanvasCommand.Present.rawValue -> {
val url = CanvasController.parseNavigateUrl(paramsJson)
canvas.navigate(url)
GatewaySession.InvokeResult.ok(null)
}
OpenClawCanvasCommand.Hide.rawValue -> GatewaySession.InvokeResult.ok(null)
OpenClawCanvasCommand.Navigate.rawValue -> {
val url = CanvasController.parseNavigateUrl(paramsJson)
canvas.navigate(url)
GatewaySession.InvokeResult.ok(null)
}
OpenClawCanvasCommand.Eval.rawValue -> {
val js =
CanvasController.parseEvalJs(paramsJson)
?: return GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: javaScript required",
)
val result =
try {
canvas.eval(js)
} catch (err: Throwable) {
return GatewaySession.InvokeResult.error(
code = "NODE_BACKGROUND_UNAVAILABLE",
message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable",
)
}
GatewaySession.InvokeResult.ok("""{"result":${result.toJsonString()}}""")
}
OpenClawCanvasCommand.Snapshot.rawValue -> {
val snapshotParams = CanvasController.parseSnapshotParams(paramsJson)
val base64 =
try {
canvas.snapshotBase64(
format = snapshotParams.format,
quality = snapshotParams.quality,
maxWidth = snapshotParams.maxWidth,
)
} catch (err: Throwable) {
return GatewaySession.InvokeResult.error(
code = "NODE_BACKGROUND_UNAVAILABLE",
message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable",
)
}
GatewaySession.InvokeResult.ok("""{"format":"${snapshotParams.format.rawValue}","base64":"$base64"}""")
}
OpenClawCanvasA2UICommand.Reset.rawValue -> {
val a2uiUrl = resolveA2uiHostUrl()
?: return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_NOT_CONFIGURED",
message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
)
val ready = ensureA2uiReady(a2uiUrl)
if (!ready) {
return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_UNAVAILABLE",
message = "A2UI host not reachable",
)
}
val res = canvas.eval(a2uiResetJS)
GatewaySession.InvokeResult.ok(res)
}
OpenClawCanvasA2UICommand.Push.rawValue, OpenClawCanvasA2UICommand.PushJSONL.rawValue -> {
val messages =
try {
decodeA2uiMessages(command, paramsJson)
} catch (err: Throwable) {
return GatewaySession.InvokeResult.error(code = "INVALID_REQUEST", message = err.message ?: "invalid A2UI payload")
}
val a2uiUrl = resolveA2uiHostUrl()
?: return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_NOT_CONFIGURED",
message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
)
val ready = ensureA2uiReady(a2uiUrl)
if (!ready) {
return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_UNAVAILABLE",
message = "A2UI host not reachable",
)
}
val js = a2uiApplyMessagesJS(messages)
val res = canvas.eval(js)
GatewaySession.InvokeResult.ok(res)
}
OpenClawCameraCommand.Snap.rawValue -> {
showCameraHud(message = "Taking photo…", kind = CameraHudKind.Photo)
triggerCameraFlash()
val res =
try {
camera.snap(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2200)
return GatewaySession.InvokeResult.error(code = code, message = message)
}
showCameraHud(message = "Photo captured", kind = CameraHudKind.Success, autoHideMs = 1600)
GatewaySession.InvokeResult.ok(res.payloadJson)
}
OpenClawCameraCommand.Clip.rawValue -> {
val includeAudio = paramsJson?.contains("\"includeAudio\":true") != false
if (includeAudio) externalAudioCaptureActive.value = true
try {
showCameraHud(message = "Recording…", kind = CameraHudKind.Recording)
val res =
try {
camera.clip(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2400)
return GatewaySession.InvokeResult.error(code = code, message = message)
}
showCameraHud(message = "Clip captured", kind = CameraHudKind.Success, autoHideMs = 1800)
GatewaySession.InvokeResult.ok(res.payloadJson)
} finally {
if (includeAudio) externalAudioCaptureActive.value = false
}
}
OpenClawLocationCommand.Get.rawValue -> {
val mode = locationMode.value
if (!isForeground.value && mode != LocationMode.Always) {
return GatewaySession.InvokeResult.error(
code = "LOCATION_BACKGROUND_UNAVAILABLE",
message = "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always",
)
}
if (!hasFineLocationPermission() && !hasCoarseLocationPermission()) {
return GatewaySession.InvokeResult.error(
code = "LOCATION_PERMISSION_REQUIRED",
message = "LOCATION_PERMISSION_REQUIRED: grant Location permission",
)
}
if (!isForeground.value && mode == LocationMode.Always && !hasBackgroundLocationPermission()) {
return GatewaySession.InvokeResult.error(
code = "LOCATION_PERMISSION_REQUIRED",
message = "LOCATION_PERMISSION_REQUIRED: enable Always in system Settings",
)
}
val (maxAgeMs, timeoutMs, desiredAccuracy) = parseLocationParams(paramsJson)
val preciseEnabled = locationPreciseEnabled.value
val accuracy =
when (desiredAccuracy) {
"precise" -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced"
"coarse" -> "coarse"
else -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced"
}
val providers =
when (accuracy) {
"precise" -> listOf(LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER)
"coarse" -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER)
else -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER)
}
try {
val payload =
location.getLocation(
desiredProviders = providers,
maxAgeMs = maxAgeMs,
timeoutMs = timeoutMs,
isPrecise = accuracy == "precise",
)
GatewaySession.InvokeResult.ok(payload.payloadJson)
} catch (err: TimeoutCancellationException) {
GatewaySession.InvokeResult.error(
code = "LOCATION_TIMEOUT",
message = "LOCATION_TIMEOUT: no fix in time",
)
} catch (err: Throwable) {
val message = err.message ?: "LOCATION_UNAVAILABLE: no fix"
GatewaySession.InvokeResult.error(code = "LOCATION_UNAVAILABLE", message = message)
}
}
OpenClawScreenCommand.Record.rawValue -> {
// Status pill mirrors screen recording state so it stays visible without overlay stacking.
_screenRecordActive.value = true
try {
val res =
try {
screenRecorder.record(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
return GatewaySession.InvokeResult.error(code = code, message = message)
}
GatewaySession.InvokeResult.ok(res.payloadJson)
} finally {
_screenRecordActive.value = false
}
}
OpenClawSmsCommand.Send.rawValue -> {
val res = sms.send(paramsJson)
if (res.ok) {
GatewaySession.InvokeResult.ok(res.payloadJson)
} else {
val error = res.error ?: "SMS_SEND_FAILED"
val idx = error.indexOf(':')
val code = if (idx > 0) error.substring(0, idx).trim() else "SMS_SEND_FAILED"
GatewaySession.InvokeResult.error(code = code, message = error)
}
}
else ->
GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: unknown command",
)
}
}
private fun triggerCameraFlash() {
// Token is used as a pulse trigger; value doesn't matter as long as it changes.
_cameraFlashToken.value = SystemClock.elapsedRealtimeNanos()
@@ -1078,194 +701,4 @@ class NodeRuntime(context: Context) {
}
}
private fun invokeErrorFromThrowable(err: Throwable): Pair<String, String> {
val raw = (err.message ?: "").trim()
if (raw.isEmpty()) return "UNAVAILABLE" to "UNAVAILABLE: camera error"
val idx = raw.indexOf(':')
if (idx <= 0) return "UNAVAILABLE" to raw
val code = raw.substring(0, idx).trim().ifEmpty { "UNAVAILABLE" }
val message = raw.substring(idx + 1).trim().ifEmpty { raw }
// Preserve full string for callers/logging, but keep the returned message human-friendly.
return code to "$code: $message"
}
private fun parseLocationParams(paramsJson: String?): Triple<Long?, Long, String?> {
if (paramsJson.isNullOrBlank()) {
return Triple(null, 10_000L, null)
}
val root =
try {
json.parseToJsonElement(paramsJson).asObjectOrNull()
} catch (_: Throwable) {
null
}
val maxAgeMs = (root?.get("maxAgeMs") as? JsonPrimitive)?.content?.toLongOrNull()
val timeoutMs =
(root?.get("timeoutMs") as? JsonPrimitive)?.content?.toLongOrNull()?.coerceIn(1_000L, 60_000L)
?: 10_000L
val desiredAccuracy =
(root?.get("desiredAccuracy") as? JsonPrimitive)?.content?.trim()?.lowercase()
return Triple(maxAgeMs, timeoutMs, desiredAccuracy)
}
private fun resolveA2uiHostUrl(): String? {
val nodeRaw = nodeSession.currentCanvasHostUrl()?.trim().orEmpty()
val operatorRaw = operatorSession.currentCanvasHostUrl()?.trim().orEmpty()
val raw = if (nodeRaw.isNotBlank()) nodeRaw else operatorRaw
if (raw.isBlank()) return null
val base = raw.trimEnd('/')
return "${base}/__openclaw__/a2ui/?platform=android"
}
private suspend fun ensureA2uiReady(a2uiUrl: String): Boolean {
try {
val already = canvas.eval(a2uiReadyCheckJS)
if (already == "true") return true
} catch (_: Throwable) {
// ignore
}
canvas.navigate(a2uiUrl)
repeat(50) {
try {
val ready = canvas.eval(a2uiReadyCheckJS)
if (ready == "true") return true
} catch (_: Throwable) {
// ignore
}
delay(120)
}
return false
}
private fun decodeA2uiMessages(command: String, paramsJson: String?): String {
val raw = paramsJson?.trim().orEmpty()
if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required")
val obj =
json.parseToJsonElement(raw) as? JsonObject
?: throw IllegalArgumentException("INVALID_REQUEST: expected object params")
val jsonlField = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty()
val hasMessagesArray = obj["messages"] is JsonArray
if (command == OpenClawCanvasA2UICommand.PushJSONL.rawValue || (!hasMessagesArray && jsonlField.isNotBlank())) {
val jsonl = jsonlField
if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required")
val messages =
jsonl
.lineSequence()
.map { it.trim() }
.filter { it.isNotBlank() }
.mapIndexed { idx, line ->
val el = json.parseToJsonElement(line)
val msg =
el as? JsonObject
?: throw IllegalArgumentException("A2UI JSONL line ${idx + 1}: expected a JSON object")
validateA2uiV0_8(msg, idx + 1)
msg
}
.toList()
return JsonArray(messages).toString()
}
val arr = obj["messages"] as? JsonArray ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required")
val out =
arr.mapIndexed { idx, el ->
val msg =
el as? JsonObject
?: throw IllegalArgumentException("A2UI messages[${idx}]: expected a JSON object")
validateA2uiV0_8(msg, idx + 1)
msg
}
return JsonArray(out).toString()
}
private fun validateA2uiV0_8(msg: JsonObject, lineNumber: Int) {
if (msg.containsKey("createSurface")) {
throw IllegalArgumentException(
"A2UI JSONL line $lineNumber: looks like A2UI v0.9 (`createSurface`). Canvas supports v0.8 messages only.",
)
}
val allowed = setOf("beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface")
val matched = msg.keys.filter { allowed.contains(it) }
if (matched.size != 1) {
val found = msg.keys.sorted().joinToString(", ")
throw IllegalArgumentException(
"A2UI JSONL line $lineNumber: expected exactly one of ${allowed.sorted().joinToString(", ")}; found: $found",
)
}
}
}
private data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D)
private const val DEFAULT_SEAM_COLOR_ARGB: Long = 0xFF4F7A9A
private const val a2uiReadyCheckJS: String =
"""
(() => {
try {
const host = globalThis.openclawA2UI;
return !!host && typeof host.applyMessages === 'function';
} catch (_) {
return false;
}
})()
"""
private const val a2uiResetJS: String =
"""
(() => {
try {
const host = globalThis.openclawA2UI;
if (!host) return { ok: false, error: "missing openclawA2UI" };
return host.reset();
} catch (e) {
return { ok: false, error: String(e?.message ?? e) };
}
})()
"""
private fun a2uiApplyMessagesJS(messagesJson: String): String {
return """
(() => {
try {
const host = globalThis.openclawA2UI;
if (!host) return { ok: false, error: "missing openclawA2UI" };
const messages = $messagesJson;
return host.applyMessages(messages);
} catch (e) {
return { ok: false, error: String(e?.message ?? e) };
}
})()
""".trimIndent()
}
private fun String.toJsonString(): String {
val escaped =
this.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\n", "\\n")
.replace("\r", "\\r")
return "\"$escaped\""
}
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
private fun JsonElement?.asStringOrNull(): String? =
when (this) {
is JsonNull -> null
is JsonPrimitive -> content
else -> null
}
private fun parseHexColorArgb(raw: String?): Long? {
val trimmed = raw?.trim().orEmpty()
if (trimmed.isEmpty()) return null
val hex = if (trimmed.startsWith("#")) trimmed.drop(1) else trimmed
if (hex.length != 6) return null
val rgb = hex.toLongOrNull(16) ?: return null
return 0xFF000000L or rgb
}

View File

@@ -71,6 +71,10 @@ class SecurePrefs(context: Context) {
MutableStateFlow(prefs.getBoolean("gateway.manual.tls", true))
val manualTls: StateFlow<Boolean> = _manualTls
private val _gatewayToken =
MutableStateFlow(prefs.getString("gateway.manual.token", "") ?: "")
val gatewayToken: StateFlow<String> = _gatewayToken
private val _lastDiscoveredStableId =
MutableStateFlow(
prefs.getString("gateway.lastDiscoveredStableID", "") ?: "",
@@ -143,12 +147,19 @@ class SecurePrefs(context: Context) {
_manualTls.value = value
}
fun setGatewayToken(value: String) {
prefs.edit { putString("gateway.manual.token", value) }
_gatewayToken.value = value
}
fun setCanvasDebugStatusEnabled(value: Boolean) {
prefs.edit { putBoolean("canvas.debugStatusEnabled", value) }
_canvasDebugStatusEnabled.value = value
}
fun loadGatewayToken(): String? {
val manual = _gatewayToken.value.trim()
if (manual.isNotEmpty()) return manual
val key = "gateway.token.${_instanceId.value}"
val stored = prefs.getString(key, null)?.trim()
return stored?.takeIf { it.isNotEmpty() }

View File

@@ -42,19 +42,45 @@ class DeviceIdentityStore(context: Context) {
fun signPayload(payload: String, identity: DeviceIdentity): String? {
return try {
// Use BC lightweight API directly — JCA provider registration is broken by R8
val privateKeyBytes = Base64.decode(identity.privateKeyPkcs8Base64, Base64.DEFAULT)
val keySpec = PKCS8EncodedKeySpec(privateKeyBytes)
val keyFactory = KeyFactory.getInstance("Ed25519")
val privateKey = keyFactory.generatePrivate(keySpec)
val signature = Signature.getInstance("Ed25519")
signature.initSign(privateKey)
signature.update(payload.toByteArray(Charsets.UTF_8))
base64UrlEncode(signature.sign())
} catch (_: Throwable) {
val pkInfo = org.bouncycastle.asn1.pkcs.PrivateKeyInfo.getInstance(privateKeyBytes)
val parsed = pkInfo.parsePrivateKey()
val rawPrivate = org.bouncycastle.asn1.DEROctetString.getInstance(parsed).octets
val privateKey = org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters(rawPrivate, 0)
val signer = org.bouncycastle.crypto.signers.Ed25519Signer()
signer.init(true, privateKey)
val payloadBytes = payload.toByteArray(Charsets.UTF_8)
signer.update(payloadBytes, 0, payloadBytes.size)
base64UrlEncode(signer.generateSignature())
} catch (e: Throwable) {
android.util.Log.e("DeviceAuth", "signPayload FAILED: ${e.javaClass.simpleName}: ${e.message}", e)
null
}
}
fun verifySelfSignature(payload: String, signatureBase64Url: String, identity: DeviceIdentity): Boolean {
return try {
val rawPublicKey = Base64.decode(identity.publicKeyRawBase64, Base64.DEFAULT)
val pubKey = org.bouncycastle.crypto.params.Ed25519PublicKeyParameters(rawPublicKey, 0)
val sigBytes = base64UrlDecode(signatureBase64Url)
val verifier = org.bouncycastle.crypto.signers.Ed25519Signer()
verifier.init(false, pubKey)
val payloadBytes = payload.toByteArray(Charsets.UTF_8)
verifier.update(payloadBytes, 0, payloadBytes.size)
verifier.verifySignature(sigBytes)
} catch (e: Throwable) {
android.util.Log.e("DeviceAuth", "self-verify exception: ${e.message}", e)
false
}
}
private fun base64UrlDecode(input: String): ByteArray {
val normalized = input.replace('-', '+').replace('_', '/')
val padded = normalized + "=".repeat((4 - normalized.length % 4) % 4)
return Base64.decode(padded, Base64.DEFAULT)
}
fun publicKeyBase64Url(identity: DeviceIdentity): String? {
return try {
val raw = Base64.decode(identity.publicKeyRawBase64, Base64.DEFAULT)
@@ -97,15 +123,21 @@ class DeviceIdentityStore(context: Context) {
}
private fun generate(): DeviceIdentity {
val keyPair = KeyPairGenerator.getInstance("Ed25519").generateKeyPair()
val spki = keyPair.public.encoded
val rawPublic = stripSpkiPrefix(spki)
// Use BC lightweight API directly to avoid JCA provider issues with R8
val kpGen = org.bouncycastle.crypto.generators.Ed25519KeyPairGenerator()
kpGen.init(org.bouncycastle.crypto.params.Ed25519KeyGenerationParameters(java.security.SecureRandom()))
val kp = kpGen.generateKeyPair()
val pubKey = kp.public as org.bouncycastle.crypto.params.Ed25519PublicKeyParameters
val privKey = kp.private as org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters
val rawPublic = pubKey.encoded // 32 bytes
val deviceId = sha256Hex(rawPublic)
val privateKey = keyPair.private.encoded
// Encode private key as PKCS8 for storage
val privKeyInfo = org.bouncycastle.crypto.util.PrivateKeyInfoFactory.createPrivateKeyInfo(privKey)
val pkcs8Bytes = privKeyInfo.encoded
return DeviceIdentity(
deviceId = deviceId,
publicKeyRawBase64 = Base64.encodeToString(rawPublic, Base64.NO_WRAP),
privateKeyPkcs8Base64 = Base64.encodeToString(privateKey, Base64.NO_WRAP),
privateKeyPkcs8Base64 = Base64.encodeToString(pkcs8Bytes, Base64.NO_WRAP),
createdAtMs = System.currentTimeMillis(),
)
}

View File

@@ -193,7 +193,9 @@ class GatewaySession(
suspend fun connect() {
val scheme = if (tls != null) "wss" else "ws"
val url = "$scheme://${endpoint.host}:${endpoint.port}"
val request = Request.Builder().url(url).build()
val httpScheme = if (tls != null) "https" else "http"
val origin = "$httpScheme://${endpoint.host}:${endpoint.port}"
val request = Request.Builder().url(url).header("Origin", origin).build()
socket = client.newWebSocket(request, Listener())
try {
connectDeferred.await()
@@ -241,6 +243,9 @@ class GatewaySession(
private fun buildClient(): OkHttpClient {
val builder = OkHttpClient.Builder()
.writeTimeout(60, java.util.concurrent.TimeUnit.SECONDS)
.readTimeout(0, java.util.concurrent.TimeUnit.SECONDS)
.pingInterval(30, java.util.concurrent.TimeUnit.SECONDS)
val tlsConfig = buildGatewayTlsConfig(tls) { fingerprint ->
onTlsFingerprint?.invoke(tls?.stableId ?: endpoint.stableId, fingerprint)
}
@@ -619,7 +624,18 @@ class GatewaySession(
val port = parsed?.port ?: -1
val scheme = parsed?.scheme?.trim().orEmpty().ifBlank { "http" }
// Detect TLS reverse proxy: endpoint on port 443, or domain-based host
val tls = endpoint.port == 443 || endpoint.host.contains(".")
// If raw URL is a non-loopback address AND we're behind TLS reverse proxy,
// fix the port (gateway sends its internal port like 18789, but we need 443 via Caddy)
if (trimmed.isNotBlank() && !isLoopbackHost(host)) {
if (tls && port > 0 && port != 443) {
// Rewrite the URL to use the reverse proxy port instead of the raw gateway port
val fixedScheme = "https"
val formattedHost = if (host.contains(":")) "[${host}]" else host
return "$fixedScheme://$formattedHost"
}
return trimmed
}
@@ -629,9 +645,14 @@ class GatewaySession(
?: endpoint.host.trim()
if (fallbackHost.isEmpty()) return trimmed.ifBlank { null }
val fallbackPort = endpoint.canvasPort ?: if (port > 0) port else 18793
// When connecting through a reverse proxy (TLS on standard port), use the
// connection endpoint's scheme and port instead of the raw canvas port.
val fallbackScheme = if (tls) "https" else scheme
// Behind reverse proxy, always use the proxy port (443), not the raw canvas port
val fallbackPort = if (tls) endpoint.port else (endpoint.canvasPort ?: endpoint.port)
val formattedHost = if (fallbackHost.contains(":")) "[${fallbackHost}]" else fallbackHost
return "$scheme://$formattedHost:$fallbackPort"
val portSuffix = if ((fallbackScheme == "https" && fallbackPort == 443) || (fallbackScheme == "http" && fallbackPort == 80)) "" else ":$fallbackPort"
return "$fallbackScheme://$formattedHost$portSuffix"
}
private fun isLoopbackHost(raw: String?): Boolean {

View File

@@ -0,0 +1,146 @@
package ai.openclaw.android.node
import ai.openclaw.android.gateway.GatewaySession
import kotlinx.coroutines.delay
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
class A2UIHandler(
private val canvas: CanvasController,
private val json: Json,
private val getNodeCanvasHostUrl: () -> String?,
private val getOperatorCanvasHostUrl: () -> String?,
) {
fun resolveA2uiHostUrl(): String? {
val nodeRaw = getNodeCanvasHostUrl()?.trim().orEmpty()
val operatorRaw = getOperatorCanvasHostUrl()?.trim().orEmpty()
val raw = if (nodeRaw.isNotBlank()) nodeRaw else operatorRaw
if (raw.isBlank()) return null
val base = raw.trimEnd('/')
return "${base}/__openclaw__/a2ui/?platform=android"
}
suspend fun ensureA2uiReady(a2uiUrl: String): Boolean {
try {
val already = canvas.eval(a2uiReadyCheckJS)
if (already == "true") return true
} catch (_: Throwable) {
// ignore
}
canvas.navigate(a2uiUrl)
repeat(50) {
try {
val ready = canvas.eval(a2uiReadyCheckJS)
if (ready == "true") return true
} catch (_: Throwable) {
// ignore
}
delay(120)
}
return false
}
fun decodeA2uiMessages(command: String, paramsJson: String?): String {
val raw = paramsJson?.trim().orEmpty()
if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required")
val obj =
json.parseToJsonElement(raw) as? JsonObject
?: throw IllegalArgumentException("INVALID_REQUEST: expected object params")
val jsonlField = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty()
val hasMessagesArray = obj["messages"] is JsonArray
if (command == "canvas.a2ui.pushJSONL" || (!hasMessagesArray && jsonlField.isNotBlank())) {
val jsonl = jsonlField
if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required")
val messages =
jsonl
.lineSequence()
.map { it.trim() }
.filter { it.isNotBlank() }
.mapIndexed { idx, line ->
val el = json.parseToJsonElement(line)
val msg =
el as? JsonObject
?: throw IllegalArgumentException("A2UI JSONL line ${idx + 1}: expected a JSON object")
validateA2uiV0_8(msg, idx + 1)
msg
}
.toList()
return JsonArray(messages).toString()
}
val arr = obj["messages"] as? JsonArray ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required")
val out =
arr.mapIndexed { idx, el ->
val msg =
el as? JsonObject
?: throw IllegalArgumentException("A2UI messages[${idx}]: expected a JSON object")
validateA2uiV0_8(msg, idx + 1)
msg
}
return JsonArray(out).toString()
}
private fun validateA2uiV0_8(msg: JsonObject, lineNumber: Int) {
if (msg.containsKey("createSurface")) {
throw IllegalArgumentException(
"A2UI JSONL line $lineNumber: looks like A2UI v0.9 (`createSurface`). Canvas supports v0.8 messages only.",
)
}
val allowed = setOf("beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface")
val matched = msg.keys.filter { allowed.contains(it) }
if (matched.size != 1) {
val found = msg.keys.sorted().joinToString(", ")
throw IllegalArgumentException(
"A2UI JSONL line $lineNumber: expected exactly one of ${allowed.sorted().joinToString(", ")}; found: $found",
)
}
}
companion object {
const val a2uiReadyCheckJS: String =
"""
(() => {
try {
const host = globalThis.openclawA2UI;
return !!host && typeof host.applyMessages === 'function';
} catch (_) {
return false;
}
})()
"""
const val a2uiResetJS: String =
"""
(() => {
try {
const host = globalThis.openclawA2UI;
if (!host) return { ok: false, error: "missing openclawA2UI" };
return host.reset();
} catch (e) {
return { ok: false, error: String(e?.message ?? e) };
}
})()
"""
fun a2uiApplyMessagesJS(messagesJson: String): String {
return """
(() => {
try {
const host = globalThis.openclawA2UI;
if (!host) return { ok: false, error: "missing openclawA2UI" };
const messages = $messagesJson;
return host.applyMessages(messages);
} catch (e) {
return { ok: false, error: String(e?.message ?? e) };
}
})()
""".trimIndent()
}
}
}

View File

@@ -0,0 +1,293 @@
package ai.openclaw.android.node
import android.app.PendingIntent
import android.content.Context
import android.content.Intent
import ai.openclaw.android.InstallResultReceiver
import ai.openclaw.android.MainActivity
import ai.openclaw.android.gateway.GatewayEndpoint
import ai.openclaw.android.gateway.GatewaySession
import java.io.File
import java.net.URI
import java.security.MessageDigest
import java.util.Locale
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put
private val SHA256_HEX = Regex("^[a-fA-F0-9]{64}$")
internal data class AppUpdateRequest(
val url: String,
val expectedSha256: String,
)
internal fun parseAppUpdateRequest(paramsJson: String?, connectedHost: String?): AppUpdateRequest {
val params =
try {
paramsJson?.let { Json.parseToJsonElement(it).jsonObject }
} catch (_: Throwable) {
throw IllegalArgumentException("params must be valid JSON")
} ?: throw IllegalArgumentException("missing 'url' parameter")
val urlRaw =
params["url"]?.jsonPrimitive?.content?.trim().orEmpty()
.ifEmpty { throw IllegalArgumentException("missing 'url' parameter") }
val sha256Raw =
params["sha256"]?.jsonPrimitive?.content?.trim().orEmpty()
.ifEmpty { throw IllegalArgumentException("missing 'sha256' parameter") }
if (!SHA256_HEX.matches(sha256Raw)) {
throw IllegalArgumentException("invalid 'sha256' parameter (expected 64 hex chars)")
}
val uri =
try {
URI(urlRaw)
} catch (_: Throwable) {
throw IllegalArgumentException("invalid 'url' parameter")
}
val scheme = uri.scheme?.lowercase(Locale.US).orEmpty()
if (scheme != "https") {
throw IllegalArgumentException("url must use https")
}
if (!uri.userInfo.isNullOrBlank()) {
throw IllegalArgumentException("url must not include credentials")
}
val host = uri.host?.lowercase(Locale.US) ?: throw IllegalArgumentException("url host required")
val connectedHostNormalized = connectedHost?.trim()?.lowercase(Locale.US).orEmpty()
if (connectedHostNormalized.isNotEmpty() && host != connectedHostNormalized) {
throw IllegalArgumentException("url host must match connected gateway host")
}
return AppUpdateRequest(
url = uri.toASCIIString(),
expectedSha256 = sha256Raw.lowercase(Locale.US),
)
}
internal fun sha256Hex(file: File): String {
val digest = MessageDigest.getInstance("SHA-256")
file.inputStream().use { input ->
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
while (true) {
val read = input.read(buffer)
if (read < 0) break
if (read == 0) continue
digest.update(buffer, 0, read)
}
}
val out = StringBuilder(64)
for (byte in digest.digest()) {
out.append(String.format(Locale.US, "%02x", byte))
}
return out.toString()
}
class AppUpdateHandler(
private val appContext: Context,
private val connectedEndpoint: () -> GatewayEndpoint?,
) {
fun handleUpdate(paramsJson: String?): GatewaySession.InvokeResult {
try {
val updateRequest =
try {
parseAppUpdateRequest(paramsJson, connectedEndpoint()?.host)
} catch (err: IllegalArgumentException) {
return GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: ${err.message ?: "invalid app.update params"}",
)
}
val url = updateRequest.url
val expectedSha256 = updateRequest.expectedSha256
android.util.Log.w("openclaw", "app.update: downloading from $url")
val notifId = 9001
val channelId = "app_update"
val notifManager = appContext.getSystemService(android.content.Context.NOTIFICATION_SERVICE) as android.app.NotificationManager
// Create notification channel (required for Android 8+)
val channel = android.app.NotificationChannel(channelId, "App Updates", android.app.NotificationManager.IMPORTANCE_LOW)
notifManager.createNotificationChannel(channel)
// PendingIntent to open the app when notification is tapped
val launchIntent = Intent(appContext, MainActivity::class.java).apply {
flags = Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP
}
val launchPi = PendingIntent.getActivity(appContext, 0, launchIntent, PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE)
// Launch download async so the invoke returns immediately
CoroutineScope(Dispatchers.IO).launch {
try {
val cacheDir = java.io.File(appContext.cacheDir, "updates")
cacheDir.mkdirs()
val file = java.io.File(cacheDir, "update.apk")
if (file.exists()) file.delete()
// Show initial progress notification
fun buildProgressNotif(progress: Int, max: Int, text: String): android.app.Notification {
return android.app.Notification.Builder(appContext, channelId)
.setSmallIcon(android.R.drawable.stat_sys_download)
.setContentTitle("OpenClaw Update")
.setContentText(text)
.setProgress(max, progress, max == 0)
.setContentIntent(launchPi)
.setOngoing(true)
.build()
}
notifManager.notify(notifId, buildProgressNotif(0, 0, "Connecting..."))
val client = okhttp3.OkHttpClient.Builder()
.connectTimeout(30, java.util.concurrent.TimeUnit.SECONDS)
.readTimeout(300, java.util.concurrent.TimeUnit.SECONDS)
.build()
val request = okhttp3.Request.Builder().url(url).build()
val response = client.newCall(request).execute()
if (!response.isSuccessful) {
notifManager.cancel(notifId)
notifManager.notify(notifId, android.app.Notification.Builder(appContext, channelId)
.setSmallIcon(android.R.drawable.stat_notify_error)
.setContentTitle("Update Failed")
.setContentIntent(launchPi)
.setContentText("HTTP ${response.code}")
.build())
return@launch
}
val contentLength = response.body?.contentLength() ?: -1L
val body = response.body ?: run {
notifManager.cancel(notifId)
return@launch
}
// Download with progress tracking
var totalBytes = 0L
var lastNotifUpdate = 0L
body.byteStream().use { input ->
file.outputStream().use { output ->
val buffer = ByteArray(8192)
while (true) {
val bytesRead = input.read(buffer)
if (bytesRead == -1) break
output.write(buffer, 0, bytesRead)
totalBytes += bytesRead
// Update notification at most every 500ms
val now = System.currentTimeMillis()
if (now - lastNotifUpdate > 500) {
lastNotifUpdate = now
if (contentLength > 0) {
val pct = ((totalBytes * 100) / contentLength).toInt()
val mb = String.format("%.1f", totalBytes / 1048576.0)
val totalMb = String.format("%.1f", contentLength / 1048576.0)
notifManager.notify(notifId, buildProgressNotif(pct, 100, "$mb / $totalMb MB ($pct%)"))
} else {
val mb = String.format("%.1f", totalBytes / 1048576.0)
notifManager.notify(notifId, buildProgressNotif(0, 0, "${mb} MB downloaded"))
}
}
}
}
}
android.util.Log.w("openclaw", "app.update: downloaded ${file.length()} bytes")
val actualSha256 = sha256Hex(file)
if (actualSha256 != expectedSha256) {
android.util.Log.e(
"openclaw",
"app.update: sha256 mismatch expected=$expectedSha256 actual=$actualSha256",
)
file.delete()
notifManager.cancel(notifId)
notifManager.notify(
notifId,
android.app.Notification.Builder(appContext, channelId)
.setSmallIcon(android.R.drawable.stat_notify_error)
.setContentTitle("Update Failed")
.setContentIntent(launchPi)
.setContentText("SHA-256 mismatch")
.build(),
)
return@launch
}
// Verify file is a valid APK (basic check: ZIP magic bytes)
val magic = file.inputStream().use { it.read().toByte() to it.read().toByte() }
if (magic.first != 0x50.toByte() || magic.second != 0x4B.toByte()) {
android.util.Log.e("openclaw", "app.update: invalid APK (bad magic: ${magic.first}, ${magic.second})")
file.delete()
notifManager.cancel(notifId)
notifManager.notify(notifId, android.app.Notification.Builder(appContext, channelId)
.setSmallIcon(android.R.drawable.stat_notify_error)
.setContentTitle("Update Failed")
.setContentIntent(launchPi)
.setContentText("Downloaded file is not a valid APK")
.build())
return@launch
}
// Use PackageInstaller session API — works from background on API 34+
// The system handles showing the install confirmation dialog
notifManager.cancel(notifId)
notifManager.notify(notifId, android.app.Notification.Builder(appContext, channelId)
.setSmallIcon(android.R.drawable.stat_sys_download_done)
.setContentTitle("Installing Update...")
.setContentIntent(launchPi)
.setContentText("${String.format("%.1f", totalBytes / 1048576.0)} MB downloaded")
.build())
val installer = appContext.packageManager.packageInstaller
val params = android.content.pm.PackageInstaller.SessionParams(
android.content.pm.PackageInstaller.SessionParams.MODE_FULL_INSTALL
)
params.setSize(file.length())
val sessionId = installer.createSession(params)
val session = installer.openSession(sessionId)
session.openWrite("openclaw-update.apk", 0, file.length()).use { out ->
file.inputStream().use { inp -> inp.copyTo(out) }
session.fsync(out)
}
// Commit with FLAG_MUTABLE PendingIntent — system requires mutable for PackageInstaller status
val callbackIntent = android.content.Intent(appContext, InstallResultReceiver::class.java)
val pi = android.app.PendingIntent.getBroadcast(
appContext, sessionId, callbackIntent,
android.app.PendingIntent.FLAG_UPDATE_CURRENT or android.app.PendingIntent.FLAG_MUTABLE
)
session.commit(pi.intentSender)
android.util.Log.w("openclaw", "app.update: PackageInstaller session committed, waiting for user confirmation")
} catch (err: Throwable) {
android.util.Log.e("openclaw", "app.update: async error", err)
notifManager.cancel(notifId)
notifManager.notify(notifId, android.app.Notification.Builder(appContext, channelId)
.setSmallIcon(android.R.drawable.stat_notify_error)
.setContentTitle("Update Failed")
.setContentIntent(launchPi)
.setContentText(err.message ?: "Unknown error")
.build())
}
}
// Return immediately — download happens in background
return GatewaySession.InvokeResult.ok(buildJsonObject {
put("status", "downloading")
put("url", url)
put("sha256", expectedSha256)
}.toString())
} catch (err: Throwable) {
android.util.Log.e("openclaw", "app.update: error", err)
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = err.message ?: "update failed")
}
}
}

View File

@@ -15,6 +15,9 @@ import androidx.camera.core.ImageCapture
import androidx.camera.core.ImageCaptureException
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.camera.video.FileOutputOptions
import androidx.camera.video.FallbackStrategy
import androidx.camera.video.Quality
import androidx.camera.video.QualitySelector
import androidx.camera.video.Recorder
import androidx.camera.video.Recording
import androidx.camera.video.VideoCapture
@@ -36,6 +39,7 @@ import kotlin.coroutines.resumeWithException
class CameraCaptureManager(private val context: Context) {
data class Payload(val payloadJson: String)
data class FilePayload(val file: File, val durationMs: Long, val hasAudio: Boolean)
@Volatile private var lifecycleOwner: LifecycleOwner? = null
@Volatile private var permissionRequester: PermissionRequester? = null
@@ -77,8 +81,8 @@ class CameraCaptureManager(private val context: Context) {
ensureCameraPermission()
val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready")
val facing = parseFacing(paramsJson) ?: "front"
val quality = (parseQuality(paramsJson) ?: 0.9).coerceIn(0.1, 1.0)
val maxWidth = parseMaxWidth(paramsJson)
val quality = (parseQuality(paramsJson) ?: 0.5).coerceIn(0.1, 1.0)
val maxWidth = parseMaxWidth(paramsJson) ?: 800
val provider = context.cameraProvider()
val capture = ImageCapture.Builder().build()
@@ -93,7 +97,7 @@ class CameraCaptureManager(private val context: Context) {
?: throw IllegalStateException("UNAVAILABLE: failed to decode captured image")
val rotated = rotateBitmapByExif(decoded, orientation)
val scaled =
if (maxWidth != null && maxWidth > 0 && rotated.width > maxWidth) {
if (maxWidth > 0 && rotated.width > maxWidth) {
val h =
(rotated.height.toDouble() * (maxWidth.toDouble() / rotated.width.toDouble()))
.toInt()
@@ -137,7 +141,7 @@ class CameraCaptureManager(private val context: Context) {
}
@SuppressLint("MissingPermission")
suspend fun clip(paramsJson: String?): Payload =
suspend fun clip(paramsJson: String?): FilePayload =
withContext(Dispatchers.Main) {
ensureCameraPermission()
val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready")
@@ -146,19 +150,49 @@ class CameraCaptureManager(private val context: Context) {
val includeAudio = parseIncludeAudio(paramsJson) ?: true
if (includeAudio) ensureMicPermission()
android.util.Log.w("CameraCaptureManager", "clip: start facing=$facing duration=$durationMs audio=$includeAudio")
val provider = context.cameraProvider()
val recorder = Recorder.Builder().build()
android.util.Log.w("CameraCaptureManager", "clip: got camera provider")
// Use LOWEST quality for smallest files over WebSocket
val recorder = Recorder.Builder()
.setQualitySelector(
QualitySelector.from(Quality.LOWEST, FallbackStrategy.lowerQualityOrHigherThan(Quality.LOWEST))
)
.build()
val videoCapture = VideoCapture.withOutput(recorder)
val selector =
if (facing == "front") CameraSelector.DEFAULT_FRONT_CAMERA else CameraSelector.DEFAULT_BACK_CAMERA
// CameraX requires a Preview use case for the camera to start producing frames;
// without it, the encoder may get no data (ERROR_NO_VALID_DATA).
val preview = androidx.camera.core.Preview.Builder().build()
// Provide a dummy SurfaceTexture so the preview pipeline activates
val surfaceTexture = android.graphics.SurfaceTexture(0)
surfaceTexture.setDefaultBufferSize(640, 480)
preview.setSurfaceProvider { request ->
val surface = android.view.Surface(surfaceTexture)
request.provideSurface(surface, context.mainExecutor()) { result ->
surface.release()
surfaceTexture.release()
}
}
provider.unbindAll()
provider.bindToLifecycle(owner, selector, videoCapture)
android.util.Log.w("CameraCaptureManager", "clip: binding preview + videoCapture to lifecycle")
val camera = provider.bindToLifecycle(owner, selector, preview, videoCapture)
android.util.Log.w("CameraCaptureManager", "clip: bound, cameraInfo=${camera.cameraInfo}")
// Give camera pipeline time to initialize before recording
android.util.Log.w("CameraCaptureManager", "clip: warming up camera 1.5s...")
kotlinx.coroutines.delay(1_500)
val file = File.createTempFile("openclaw-clip-", ".mp4")
val outputOptions = FileOutputOptions.Builder(file).build()
val finalized = kotlinx.coroutines.CompletableDeferred<VideoRecordEvent.Finalize>()
android.util.Log.w("CameraCaptureManager", "clip: starting recording to ${file.absolutePath}")
val recording: Recording =
videoCapture.output
.prepareRecording(context, outputOptions)
@@ -166,35 +200,49 @@ class CameraCaptureManager(private val context: Context) {
if (includeAudio) withAudioEnabled()
}
.start(context.mainExecutor()) { event ->
android.util.Log.w("CameraCaptureManager", "clip: event ${event.javaClass.simpleName}")
if (event is VideoRecordEvent.Status) {
android.util.Log.w("CameraCaptureManager", "clip: recording status update")
}
if (event is VideoRecordEvent.Finalize) {
android.util.Log.w("CameraCaptureManager", "clip: finalize hasError=${event.hasError()} error=${event.error} cause=${event.cause}")
finalized.complete(event)
}
}
android.util.Log.w("CameraCaptureManager", "clip: recording started, delaying ${durationMs}ms")
try {
kotlinx.coroutines.delay(durationMs.toLong())
} finally {
android.util.Log.w("CameraCaptureManager", "clip: stopping recording")
recording.stop()
}
val finalizeEvent =
try {
withTimeout(10_000) { finalized.await() }
withTimeout(15_000) { finalized.await() }
} catch (err: Throwable) {
file.delete()
android.util.Log.e("CameraCaptureManager", "clip: finalize timed out", err)
withContext(Dispatchers.IO) { file.delete() }
provider.unbindAll()
throw IllegalStateException("UNAVAILABLE: camera clip finalize timed out")
}
if (finalizeEvent.hasError()) {
file.delete()
throw IllegalStateException("UNAVAILABLE: camera clip failed")
android.util.Log.e("CameraCaptureManager", "clip: FAILED error=${finalizeEvent.error}, cause=${finalizeEvent.cause}", finalizeEvent.cause)
// Check file size for debugging
val fileSize = withContext(Dispatchers.IO) { if (file.exists()) file.length() else -1 }
android.util.Log.e("CameraCaptureManager", "clip: file exists=${file.exists()} size=$fileSize")
withContext(Dispatchers.IO) { file.delete() }
provider.unbindAll()
throw IllegalStateException("UNAVAILABLE: camera clip failed (error=${finalizeEvent.error})")
}
val bytes = file.readBytes()
file.delete()
val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP)
Payload(
"""{"format":"mp4","base64":"$base64","durationMs":$durationMs,"hasAudio":${includeAudio}}""",
)
val fileSize = withContext(Dispatchers.IO) { file.length() }
android.util.Log.w("CameraCaptureManager", "clip: SUCCESS file size=$fileSize")
provider.unbindAll()
FilePayload(file = file, durationMs = durationMs.toLong(), hasAudio = includeAudio)
}
private fun rotateBitmapByExif(bitmap: Bitmap, orientation: Int): Bitmap {

View File

@@ -0,0 +1,157 @@
package ai.openclaw.android.node
import android.content.Context
import ai.openclaw.android.CameraHudKind
import ai.openclaw.android.BuildConfig
import ai.openclaw.android.SecurePrefs
import ai.openclaw.android.gateway.GatewayEndpoint
import ai.openclaw.android.gateway.GatewaySession
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.withContext
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.RequestBody.Companion.asRequestBody
class CameraHandler(
private val appContext: Context,
private val camera: CameraCaptureManager,
private val prefs: SecurePrefs,
private val connectedEndpoint: () -> GatewayEndpoint?,
private val externalAudioCaptureActive: MutableStateFlow<Boolean>,
private val showCameraHud: (message: String, kind: CameraHudKind, autoHideMs: Long?) -> Unit,
private val triggerCameraFlash: () -> Unit,
private val invokeErrorFromThrowable: (err: Throwable) -> Pair<String, String>,
) {
suspend fun handleSnap(paramsJson: String?): GatewaySession.InvokeResult {
val logFile = if (BuildConfig.DEBUG) java.io.File(appContext.cacheDir, "camera_debug.log") else null
fun camLog(msg: String) {
if (!BuildConfig.DEBUG) return
val ts = java.text.SimpleDateFormat("HH:mm:ss.SSS", java.util.Locale.US).format(java.util.Date())
logFile?.appendText("[$ts] $msg\n")
android.util.Log.w("openclaw", "camera.snap: $msg")
}
try {
logFile?.writeText("") // clear
camLog("starting, params=$paramsJson")
camLog("calling showCameraHud")
showCameraHud("Taking photo…", CameraHudKind.Photo, null)
camLog("calling triggerCameraFlash")
triggerCameraFlash()
val res =
try {
camLog("calling camera.snap()")
val r = camera.snap(paramsJson)
camLog("success, payload size=${r.payloadJson.length}")
r
} catch (err: Throwable) {
camLog("inner error: ${err::class.java.simpleName}: ${err.message}")
camLog("stack: ${err.stackTraceToString().take(2000)}")
val (code, message) = invokeErrorFromThrowable(err)
showCameraHud(message, CameraHudKind.Error, 2200)
return GatewaySession.InvokeResult.error(code = code, message = message)
}
camLog("returning result")
showCameraHud("Photo captured", CameraHudKind.Success, 1600)
return GatewaySession.InvokeResult.ok(res.payloadJson)
} catch (err: Throwable) {
camLog("outer error: ${err::class.java.simpleName}: ${err.message}")
camLog("stack: ${err.stackTraceToString().take(2000)}")
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = err.message ?: "camera snap failed")
}
}
suspend fun handleClip(paramsJson: String?): GatewaySession.InvokeResult {
val clipLogFile = if (BuildConfig.DEBUG) java.io.File(appContext.cacheDir, "camera_debug.log") else null
fun clipLog(msg: String) {
if (!BuildConfig.DEBUG) return
val ts = java.text.SimpleDateFormat("HH:mm:ss.SSS", java.util.Locale.US).format(java.util.Date())
clipLogFile?.appendText("[CLIP $ts] $msg\n")
android.util.Log.w("openclaw", "camera.clip: $msg")
}
val includeAudio = paramsJson?.contains("\"includeAudio\":true") != false
if (includeAudio) externalAudioCaptureActive.value = true
try {
clipLogFile?.writeText("") // clear
clipLog("starting, params=$paramsJson includeAudio=$includeAudio")
clipLog("calling showCameraHud")
showCameraHud("Recording…", CameraHudKind.Recording, null)
val filePayload =
try {
clipLog("calling camera.clip()")
val r = camera.clip(paramsJson)
clipLog("success, file size=${r.file.length()}")
r
} catch (err: Throwable) {
clipLog("inner error: ${err::class.java.simpleName}: ${err.message}")
clipLog("stack: ${err.stackTraceToString().take(2000)}")
val (code, message) = invokeErrorFromThrowable(err)
showCameraHud(message, CameraHudKind.Error, 2400)
return GatewaySession.InvokeResult.error(code = code, message = message)
}
// Upload file via HTTP instead of base64 through WebSocket
clipLog("uploading via HTTP...")
val uploadUrl = try {
withContext(Dispatchers.IO) {
val ep = connectedEndpoint()
val gatewayHost = if (ep != null) {
val isHttps = ep.tlsEnabled || ep.port == 443
if (!isHttps) {
clipLog("refusing to upload over plain HTTP — bearer token would be exposed; falling back to base64")
throw Exception("HTTPS required for upload (bearer token protection)")
}
if (ep.port == 443) "https://${ep.host}" else "https://${ep.host}:${ep.port}"
} else {
clipLog("error: no gateway endpoint connected, cannot upload")
throw Exception("no gateway endpoint connected")
}
val token = prefs.loadGatewayToken() ?: ""
val client = okhttp3.OkHttpClient.Builder()
.connectTimeout(10, java.util.concurrent.TimeUnit.SECONDS)
.writeTimeout(120, java.util.concurrent.TimeUnit.SECONDS)
.readTimeout(30, java.util.concurrent.TimeUnit.SECONDS)
.build()
val body = filePayload.file.asRequestBody("video/mp4".toMediaType())
val req = okhttp3.Request.Builder()
.url("$gatewayHost/upload/clip.mp4")
.put(body)
.header("Authorization", "Bearer $token")
.build()
clipLog("uploading ${filePayload.file.length()} bytes to $gatewayHost/upload/clip.mp4")
val resp = client.newCall(req).execute()
val respBody = resp.body?.string() ?: ""
clipLog("upload response: ${resp.code} $respBody")
filePayload.file.delete()
if (!resp.isSuccessful) throw Exception("upload failed: HTTP ${resp.code}")
// Parse URL from response
val urlMatch = Regex("\"url\":\"([^\"]+)\"").find(respBody)
urlMatch?.groupValues?.get(1) ?: throw Exception("no url in response: $respBody")
}
} catch (err: Throwable) {
clipLog("upload failed: ${err.message}, falling back to base64")
// Fallback to base64 if upload fails
val bytes = withContext(Dispatchers.IO) {
val b = filePayload.file.readBytes()
filePayload.file.delete()
b
}
val base64 = android.util.Base64.encodeToString(bytes, android.util.Base64.NO_WRAP)
showCameraHud("Clip captured", CameraHudKind.Success, 1800)
return GatewaySession.InvokeResult.ok(
"""{"format":"mp4","base64":"$base64","durationMs":${filePayload.durationMs},"hasAudio":${filePayload.hasAudio}}"""
)
}
clipLog("returning URL result: $uploadUrl")
showCameraHud("Clip captured", CameraHudKind.Success, 1800)
return GatewaySession.InvokeResult.ok(
"""{"format":"mp4","url":"$uploadUrl","durationMs":${filePayload.durationMs},"hasAudio":${filePayload.hasAudio}}"""
)
} catch (err: Throwable) {
clipLog("outer error: ${err::class.java.simpleName}: ${err.message}")
clipLog("stack: ${err.stackTraceToString().take(2000)}")
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = err.message ?: "camera clip failed")
} finally {
if (includeAudio) externalAudioCaptureActive.value = false
}
}
}

View File

@@ -0,0 +1,166 @@
package ai.openclaw.android.node
import android.os.Build
import ai.openclaw.android.BuildConfig
import ai.openclaw.android.SecurePrefs
import ai.openclaw.android.gateway.GatewayClientInfo
import ai.openclaw.android.gateway.GatewayConnectOptions
import ai.openclaw.android.gateway.GatewayEndpoint
import ai.openclaw.android.gateway.GatewayTlsParams
import ai.openclaw.android.protocol.OpenClawCanvasA2UICommand
import ai.openclaw.android.protocol.OpenClawCanvasCommand
import ai.openclaw.android.protocol.OpenClawCameraCommand
import ai.openclaw.android.protocol.OpenClawLocationCommand
import ai.openclaw.android.protocol.OpenClawScreenCommand
import ai.openclaw.android.protocol.OpenClawSmsCommand
import ai.openclaw.android.protocol.OpenClawCapability
import ai.openclaw.android.LocationMode
import ai.openclaw.android.VoiceWakeMode
class ConnectionManager(
private val prefs: SecurePrefs,
private val cameraEnabled: () -> Boolean,
private val locationMode: () -> LocationMode,
private val voiceWakeMode: () -> VoiceWakeMode,
private val smsAvailable: () -> Boolean,
private val hasRecordAudioPermission: () -> Boolean,
private val manualTls: () -> Boolean,
) {
fun buildInvokeCommands(): List<String> =
buildList {
add(OpenClawCanvasCommand.Present.rawValue)
add(OpenClawCanvasCommand.Hide.rawValue)
add(OpenClawCanvasCommand.Navigate.rawValue)
add(OpenClawCanvasCommand.Eval.rawValue)
add(OpenClawCanvasCommand.Snapshot.rawValue)
add(OpenClawCanvasA2UICommand.Push.rawValue)
add(OpenClawCanvasA2UICommand.PushJSONL.rawValue)
add(OpenClawCanvasA2UICommand.Reset.rawValue)
add(OpenClawScreenCommand.Record.rawValue)
if (cameraEnabled()) {
add(OpenClawCameraCommand.Snap.rawValue)
add(OpenClawCameraCommand.Clip.rawValue)
}
if (locationMode() != LocationMode.Off) {
add(OpenClawLocationCommand.Get.rawValue)
}
if (smsAvailable()) {
add(OpenClawSmsCommand.Send.rawValue)
}
if (BuildConfig.DEBUG) {
add("debug.logs")
add("debug.ed25519")
}
add("app.update")
}
fun buildCapabilities(): List<String> =
buildList {
add(OpenClawCapability.Canvas.rawValue)
add(OpenClawCapability.Screen.rawValue)
if (cameraEnabled()) add(OpenClawCapability.Camera.rawValue)
if (smsAvailable()) add(OpenClawCapability.Sms.rawValue)
if (voiceWakeMode() != VoiceWakeMode.Off && hasRecordAudioPermission()) {
add(OpenClawCapability.VoiceWake.rawValue)
}
if (locationMode() != LocationMode.Off) {
add(OpenClawCapability.Location.rawValue)
}
}
fun resolvedVersionName(): String {
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
return if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
"$versionName-dev"
} else {
versionName
}
}
fun resolveModelIdentifier(): String? {
return listOfNotNull(Build.MANUFACTURER, Build.MODEL)
.joinToString(" ")
.trim()
.ifEmpty { null }
}
fun buildUserAgent(): String {
val version = resolvedVersionName()
val release = Build.VERSION.RELEASE?.trim().orEmpty()
val releaseLabel = if (release.isEmpty()) "unknown" else release
return "OpenClawAndroid/$version (Android $releaseLabel; SDK ${Build.VERSION.SDK_INT})"
}
fun buildClientInfo(clientId: String, clientMode: String): GatewayClientInfo {
return GatewayClientInfo(
id = clientId,
displayName = prefs.displayName.value,
version = resolvedVersionName(),
platform = "android",
mode = clientMode,
instanceId = prefs.instanceId.value,
deviceFamily = "Android",
modelIdentifier = resolveModelIdentifier(),
)
}
fun buildNodeConnectOptions(): GatewayConnectOptions {
return GatewayConnectOptions(
role = "node",
scopes = emptyList(),
caps = buildCapabilities(),
commands = buildInvokeCommands(),
permissions = emptyMap(),
client = buildClientInfo(clientId = "openclaw-android", clientMode = "node"),
userAgent = buildUserAgent(),
)
}
fun buildOperatorConnectOptions(): GatewayConnectOptions {
return GatewayConnectOptions(
role = "operator",
scopes = emptyList(),
caps = emptyList(),
commands = emptyList(),
permissions = emptyMap(),
client = buildClientInfo(clientId = "openclaw-control-ui", clientMode = "ui"),
userAgent = buildUserAgent(),
)
}
fun resolveTlsParams(endpoint: GatewayEndpoint): GatewayTlsParams? {
val stored = prefs.loadGatewayTlsFingerprint(endpoint.stableId)
val hinted = endpoint.tlsEnabled || !endpoint.tlsFingerprintSha256.isNullOrBlank()
val manual = endpoint.stableId.startsWith("manual|")
if (manual) {
if (!manualTls()) return null
return GatewayTlsParams(
required = true,
expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored,
allowTOFU = stored == null,
stableId = endpoint.stableId,
)
}
if (hinted) {
return GatewayTlsParams(
required = true,
expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored,
allowTOFU = stored == null,
stableId = endpoint.stableId,
)
}
if (!stored.isNullOrBlank()) {
return GatewayTlsParams(
required = true,
expectedFingerprint = stored,
allowTOFU = false,
stableId = endpoint.stableId,
)
}
return null
}
}

View File

@@ -0,0 +1,117 @@
package ai.openclaw.android.node
import android.content.Context
import ai.openclaw.android.BuildConfig
import ai.openclaw.android.gateway.DeviceIdentityStore
import ai.openclaw.android.gateway.GatewaySession
import kotlinx.serialization.json.JsonPrimitive
class DebugHandler(
private val appContext: Context,
private val identityStore: DeviceIdentityStore,
) {
fun handleEd25519(): GatewaySession.InvokeResult {
if (!BuildConfig.DEBUG) {
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = "debug commands are disabled in release builds")
}
// Self-test Ed25519 signing and return diagnostic info
try {
val identity = identityStore.loadOrCreate()
val testPayload = "test|${identity.deviceId}|${System.currentTimeMillis()}"
val results = mutableListOf<String>()
results.add("deviceId: ${identity.deviceId}")
results.add("publicKeyRawBase64: ${identity.publicKeyRawBase64.take(20)}...")
results.add("privateKeyPkcs8Base64: ${identity.privateKeyPkcs8Base64.take(20)}...")
// Test publicKeyBase64Url
val pubKeyUrl = identityStore.publicKeyBase64Url(identity)
results.add("publicKeyBase64Url: ${pubKeyUrl ?: "NULL (FAILED)"}")
// Test signing
val signature = identityStore.signPayload(testPayload, identity)
results.add("signPayload: ${if (signature != null) "${signature.take(20)}... (OK)" else "NULL (FAILED)"}")
// Test self-verify
if (signature != null) {
val verifyOk = identityStore.verifySelfSignature(testPayload, signature, identity)
results.add("verifySelfSignature: $verifyOk")
}
// Check available providers
val providers = java.security.Security.getProviders()
val ed25519Providers = providers.filter { p ->
p.services.any { s -> s.algorithm.contains("Ed25519", ignoreCase = true) }
}
results.add("Ed25519 providers: ${ed25519Providers.map { "${it.name} v${it.version}" }}")
results.add("Provider order: ${providers.take(5).map { it.name }}")
// Test KeyFactory directly
try {
val kf = java.security.KeyFactory.getInstance("Ed25519")
results.add("KeyFactory.Ed25519: ${kf.provider.name} (OK)")
} catch (e: Throwable) {
results.add("KeyFactory.Ed25519: FAILED - ${e.javaClass.simpleName}: ${e.message}")
}
// Test Signature directly
try {
val sig = java.security.Signature.getInstance("Ed25519")
results.add("Signature.Ed25519: ${sig.provider.name} (OK)")
} catch (e: Throwable) {
results.add("Signature.Ed25519: FAILED - ${e.javaClass.simpleName}: ${e.message}")
}
return GatewaySession.InvokeResult.ok("""{"diagnostics":"${results.joinToString("\\n").replace("\"", "\\\"")}"}"""")
} catch (e: Throwable) {
return GatewaySession.InvokeResult.error(code = "ED25519_TEST_FAILED", message = "${e.javaClass.simpleName}: ${e.message}\n${e.stackTraceToString().take(500)}")
}
}
fun handleLogs(): GatewaySession.InvokeResult {
if (!BuildConfig.DEBUG) {
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = "debug commands are disabled in release builds")
}
val pid = android.os.Process.myPid()
val rt = Runtime.getRuntime()
val info = "v6 pid=$pid thread=${Thread.currentThread().name} free=${rt.freeMemory()/1024}K total=${rt.totalMemory()/1024}K max=${rt.maxMemory()/1024}K uptime=${android.os.SystemClock.elapsedRealtime()/1000}s sdk=${android.os.Build.VERSION.SDK_INT} device=${android.os.Build.MODEL}\n"
// Run logcat on current dispatcher thread (no withContext) with file redirect
val logResult = try {
val tmpFile = java.io.File(appContext.cacheDir, "debug_logs.txt")
if (tmpFile.exists()) tmpFile.delete()
val pb = ProcessBuilder("logcat", "-d", "-t", "200", "--pid=$pid")
pb.redirectOutput(tmpFile)
pb.redirectErrorStream(true)
val proc = pb.start()
val finished = proc.waitFor(4, java.util.concurrent.TimeUnit.SECONDS)
if (!finished) proc.destroyForcibly()
val raw = if (tmpFile.exists() && tmpFile.length() > 0) {
tmpFile.readText().take(128000)
} else {
"(no output, finished=$finished, exists=${tmpFile.exists()})"
}
tmpFile.delete()
val spamPatterns = listOf("setRequestedFrameRate", "I View :", "BLASTBufferQueue", "VRI[Pop-Up",
"InsetsController:", "VRI[MainActivity", "InsetsSource:", "handleResized", "ProfileInstaller",
"I VRI[", "onStateChanged: host=", "D StrictMode:", "E StrictMode:", "ImeFocusController",
"InputTransport", "IncorrectContextUseViolation")
val sb = StringBuilder()
for (line in raw.lineSequence()) {
if (line.isBlank()) continue
if (spamPatterns.any { line.contains(it) }) continue
if (sb.length + line.length > 16000) { sb.append("\n(truncated)"); break }
if (sb.isNotEmpty()) sb.append('\n')
sb.append(line)
}
sb.toString().ifEmpty { "(all ${raw.lines().size} lines filtered as spam)" }
} catch (e: Throwable) {
"(logcat error: ${e::class.java.simpleName}: ${e.message})"
}
// Also include camera debug log if it exists
val camLogFile = java.io.File(appContext.cacheDir, "camera_debug.log")
val camLog = if (camLogFile.exists() && camLogFile.length() > 0) {
"\n--- camera_debug.log ---\n" + camLogFile.readText().take(4000)
} else ""
return GatewaySession.InvokeResult.ok("""{"logs":${JsonPrimitive(info + logResult + camLog)}}""")
}
}

View File

@@ -0,0 +1,71 @@
package ai.openclaw.android.node
import ai.openclaw.android.SecurePrefs
import ai.openclaw.android.gateway.GatewaySession
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
class GatewayEventHandler(
private val scope: CoroutineScope,
private val prefs: SecurePrefs,
private val json: Json,
private val operatorSession: GatewaySession,
private val isConnected: () -> Boolean,
) {
private var suppressWakeWordsSync = false
private var wakeWordsSyncJob: Job? = null
fun applyWakeWordsFromGateway(words: List<String>) {
suppressWakeWordsSync = true
prefs.setWakeWords(words)
suppressWakeWordsSync = false
}
fun scheduleWakeWordsSyncIfNeeded() {
if (suppressWakeWordsSync) return
if (!isConnected()) return
val snapshot = prefs.wakeWords.value
wakeWordsSyncJob?.cancel()
wakeWordsSyncJob =
scope.launch {
delay(650)
val jsonList = snapshot.joinToString(separator = ",") { it.toJsonString() }
val params = """{"triggers":[$jsonList]}"""
try {
operatorSession.request("voicewake.set", params)
} catch (_: Throwable) {
// ignore
}
}
}
suspend fun refreshWakeWordsFromGateway() {
if (!isConnected()) return
try {
val res = operatorSession.request("voicewake.get", "{}")
val payload = json.parseToJsonElement(res).asObjectOrNull() ?: return
val array = payload["triggers"] as? JsonArray ?: return
val triggers = array.mapNotNull { it.asStringOrNull() }
applyWakeWordsFromGateway(triggers)
} catch (_: Throwable) {
// ignore
}
}
fun handleVoiceWakeChangedEvent(payloadJson: String?) {
if (payloadJson.isNullOrBlank()) return
try {
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
val array = payload["triggers"] as? JsonArray ?: return
val triggers = array.mapNotNull { it.asStringOrNull() }
applyWakeWordsFromGateway(triggers)
} catch (_: Throwable) {
// ignore
}
}
}

View File

@@ -0,0 +1,176 @@
package ai.openclaw.android.node
import ai.openclaw.android.gateway.GatewaySession
import ai.openclaw.android.protocol.OpenClawCanvasA2UICommand
import ai.openclaw.android.protocol.OpenClawCanvasCommand
import ai.openclaw.android.protocol.OpenClawCameraCommand
import ai.openclaw.android.protocol.OpenClawLocationCommand
import ai.openclaw.android.protocol.OpenClawScreenCommand
import ai.openclaw.android.protocol.OpenClawSmsCommand
class InvokeDispatcher(
private val canvas: CanvasController,
private val cameraHandler: CameraHandler,
private val locationHandler: LocationHandler,
private val screenHandler: ScreenHandler,
private val smsHandler: SmsHandler,
private val a2uiHandler: A2UIHandler,
private val debugHandler: DebugHandler,
private val appUpdateHandler: AppUpdateHandler,
private val isForeground: () -> Boolean,
private val cameraEnabled: () -> Boolean,
private val locationEnabled: () -> Boolean,
) {
suspend fun handleInvoke(command: String, paramsJson: String?): GatewaySession.InvokeResult {
// Check foreground requirement for canvas/camera/screen commands
if (
command.startsWith(OpenClawCanvasCommand.NamespacePrefix) ||
command.startsWith(OpenClawCanvasA2UICommand.NamespacePrefix) ||
command.startsWith(OpenClawCameraCommand.NamespacePrefix) ||
command.startsWith(OpenClawScreenCommand.NamespacePrefix)
) {
if (!isForeground()) {
return GatewaySession.InvokeResult.error(
code = "NODE_BACKGROUND_UNAVAILABLE",
message = "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground",
)
}
}
// Check camera enabled
if (command.startsWith(OpenClawCameraCommand.NamespacePrefix) && !cameraEnabled()) {
return GatewaySession.InvokeResult.error(
code = "CAMERA_DISABLED",
message = "CAMERA_DISABLED: enable Camera in Settings",
)
}
// Check location enabled
if (command.startsWith(OpenClawLocationCommand.NamespacePrefix) && !locationEnabled()) {
return GatewaySession.InvokeResult.error(
code = "LOCATION_DISABLED",
message = "LOCATION_DISABLED: enable Location in Settings",
)
}
return when (command) {
// Canvas commands
OpenClawCanvasCommand.Present.rawValue -> {
val url = CanvasController.parseNavigateUrl(paramsJson)
canvas.navigate(url)
GatewaySession.InvokeResult.ok(null)
}
OpenClawCanvasCommand.Hide.rawValue -> GatewaySession.InvokeResult.ok(null)
OpenClawCanvasCommand.Navigate.rawValue -> {
val url = CanvasController.parseNavigateUrl(paramsJson)
canvas.navigate(url)
GatewaySession.InvokeResult.ok(null)
}
OpenClawCanvasCommand.Eval.rawValue -> {
val js =
CanvasController.parseEvalJs(paramsJson)
?: return GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: javaScript required",
)
val result =
try {
canvas.eval(js)
} catch (err: Throwable) {
return GatewaySession.InvokeResult.error(
code = "NODE_BACKGROUND_UNAVAILABLE",
message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable",
)
}
GatewaySession.InvokeResult.ok("""{"result":${result.toJsonString()}}""")
}
OpenClawCanvasCommand.Snapshot.rawValue -> {
val snapshotParams = CanvasController.parseSnapshotParams(paramsJson)
val base64 =
try {
canvas.snapshotBase64(
format = snapshotParams.format,
quality = snapshotParams.quality,
maxWidth = snapshotParams.maxWidth,
)
} catch (err: Throwable) {
return GatewaySession.InvokeResult.error(
code = "NODE_BACKGROUND_UNAVAILABLE",
message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable",
)
}
GatewaySession.InvokeResult.ok("""{"format":"${snapshotParams.format.rawValue}","base64":"$base64"}""")
}
// A2UI commands
OpenClawCanvasA2UICommand.Reset.rawValue -> {
val a2uiUrl = a2uiHandler.resolveA2uiHostUrl()
?: return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_NOT_CONFIGURED",
message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
)
val ready = a2uiHandler.ensureA2uiReady(a2uiUrl)
if (!ready) {
return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_UNAVAILABLE",
message = "A2UI host not reachable",
)
}
val res = canvas.eval(A2UIHandler.a2uiResetJS)
GatewaySession.InvokeResult.ok(res)
}
OpenClawCanvasA2UICommand.Push.rawValue, OpenClawCanvasA2UICommand.PushJSONL.rawValue -> {
val messages =
try {
a2uiHandler.decodeA2uiMessages(command, paramsJson)
} catch (err: Throwable) {
return GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = err.message ?: "invalid A2UI payload"
)
}
val a2uiUrl = a2uiHandler.resolveA2uiHostUrl()
?: return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_NOT_CONFIGURED",
message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
)
val ready = a2uiHandler.ensureA2uiReady(a2uiUrl)
if (!ready) {
return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_UNAVAILABLE",
message = "A2UI host not reachable",
)
}
val js = A2UIHandler.a2uiApplyMessagesJS(messages)
val res = canvas.eval(js)
GatewaySession.InvokeResult.ok(res)
}
// Camera commands
OpenClawCameraCommand.Snap.rawValue -> cameraHandler.handleSnap(paramsJson)
OpenClawCameraCommand.Clip.rawValue -> cameraHandler.handleClip(paramsJson)
// Location command
OpenClawLocationCommand.Get.rawValue -> locationHandler.handleLocationGet(paramsJson)
// Screen command
OpenClawScreenCommand.Record.rawValue -> screenHandler.handleScreenRecord(paramsJson)
// SMS command
OpenClawSmsCommand.Send.rawValue -> smsHandler.handleSmsSend(paramsJson)
// Debug commands
"debug.ed25519" -> debugHandler.handleEd25519()
"debug.logs" -> debugHandler.handleLogs()
// App update
"app.update" -> appUpdateHandler.handleUpdate(paramsJson)
else ->
GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: unknown command",
)
}
}
}

View File

@@ -0,0 +1,116 @@
package ai.openclaw.android.node
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.location.LocationManager
import androidx.core.content.ContextCompat
import ai.openclaw.android.LocationMode
import ai.openclaw.android.gateway.GatewaySession
import kotlinx.coroutines.TimeoutCancellationException
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
class LocationHandler(
private val appContext: Context,
private val location: LocationCaptureManager,
private val json: Json,
private val isForeground: () -> Boolean,
private val locationMode: () -> LocationMode,
private val locationPreciseEnabled: () -> Boolean,
) {
fun hasFineLocationPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_FINE_LOCATION) ==
PackageManager.PERMISSION_GRANTED
)
}
fun hasCoarseLocationPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_COARSE_LOCATION) ==
PackageManager.PERMISSION_GRANTED
)
}
fun hasBackgroundLocationPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_BACKGROUND_LOCATION) ==
PackageManager.PERMISSION_GRANTED
)
}
suspend fun handleLocationGet(paramsJson: String?): GatewaySession.InvokeResult {
val mode = locationMode()
if (!isForeground() && mode != LocationMode.Always) {
return GatewaySession.InvokeResult.error(
code = "LOCATION_BACKGROUND_UNAVAILABLE",
message = "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always",
)
}
if (!hasFineLocationPermission() && !hasCoarseLocationPermission()) {
return GatewaySession.InvokeResult.error(
code = "LOCATION_PERMISSION_REQUIRED",
message = "LOCATION_PERMISSION_REQUIRED: grant Location permission",
)
}
if (!isForeground() && mode == LocationMode.Always && !hasBackgroundLocationPermission()) {
return GatewaySession.InvokeResult.error(
code = "LOCATION_PERMISSION_REQUIRED",
message = "LOCATION_PERMISSION_REQUIRED: enable Always in system Settings",
)
}
val (maxAgeMs, timeoutMs, desiredAccuracy) = parseLocationParams(paramsJson)
val preciseEnabled = locationPreciseEnabled()
val accuracy =
when (desiredAccuracy) {
"precise" -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced"
"coarse" -> "coarse"
else -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced"
}
val providers =
when (accuracy) {
"precise" -> listOf(LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER)
"coarse" -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER)
else -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER)
}
try {
val payload =
location.getLocation(
desiredProviders = providers,
maxAgeMs = maxAgeMs,
timeoutMs = timeoutMs,
isPrecise = accuracy == "precise",
)
return GatewaySession.InvokeResult.ok(payload.payloadJson)
} catch (err: TimeoutCancellationException) {
return GatewaySession.InvokeResult.error(
code = "LOCATION_TIMEOUT",
message = "LOCATION_TIMEOUT: no fix in time",
)
} catch (err: Throwable) {
val message = err.message ?: "LOCATION_UNAVAILABLE: no fix"
return GatewaySession.InvokeResult.error(code = "LOCATION_UNAVAILABLE", message = message)
}
}
private fun parseLocationParams(paramsJson: String?): Triple<Long?, Long, String?> {
if (paramsJson.isNullOrBlank()) {
return Triple(null, 10_000L, null)
}
val root =
try {
json.parseToJsonElement(paramsJson).asObjectOrNull()
} catch (_: Throwable) {
null
}
val maxAgeMs = (root?.get("maxAgeMs") as? JsonPrimitive)?.content?.toLongOrNull()
val timeoutMs =
(root?.get("timeoutMs") as? JsonPrimitive)?.content?.toLongOrNull()?.coerceIn(1_000L, 60_000L)
?: 10_000L
val desiredAccuracy =
(root?.get("desiredAccuracy") as? JsonPrimitive)?.content?.trim()?.lowercase()
return Triple(maxAgeMs, timeoutMs, desiredAccuracy)
}
}

View File

@@ -0,0 +1,57 @@
package ai.openclaw.android.node
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
const val DEFAULT_SEAM_COLOR_ARGB: Long = 0xFF4F7A9A
data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D)
fun String.toJsonString(): String {
val escaped =
this.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\n", "\\n")
.replace("\r", "\\r")
return "\"$escaped\""
}
fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
fun JsonElement?.asStringOrNull(): String? =
when (this) {
is JsonNull -> null
is JsonPrimitive -> content
else -> null
}
fun parseHexColorArgb(raw: String?): Long? {
val trimmed = raw?.trim().orEmpty()
if (trimmed.isEmpty()) return null
val hex = if (trimmed.startsWith("#")) trimmed.drop(1) else trimmed
if (hex.length != 6) return null
val rgb = hex.toLongOrNull(16) ?: return null
return 0xFF000000L or rgb
}
fun invokeErrorFromThrowable(err: Throwable): Pair<String, String> {
val raw = (err.message ?: "").trim()
if (raw.isEmpty()) return "UNAVAILABLE" to "UNAVAILABLE: error"
val idx = raw.indexOf(':')
if (idx <= 0) return "UNAVAILABLE" to raw
val code = raw.substring(0, idx).trim().ifEmpty { "UNAVAILABLE" }
val message = raw.substring(idx + 1).trim().ifEmpty { raw }
return code to "$code: $message"
}
fun normalizeMainKey(raw: String?): String? {
val trimmed = raw?.trim().orEmpty()
return if (trimmed.isEmpty()) null else trimmed
}
fun isCanonicalMainSessionKey(key: String): Boolean {
return key == "main"
}

View File

@@ -0,0 +1,25 @@
package ai.openclaw.android.node
import ai.openclaw.android.gateway.GatewaySession
class ScreenHandler(
private val screenRecorder: ScreenRecordManager,
private val setScreenRecordActive: (Boolean) -> Unit,
private val invokeErrorFromThrowable: (Throwable) -> Pair<String, String>,
) {
suspend fun handleScreenRecord(paramsJson: String?): GatewaySession.InvokeResult {
setScreenRecordActive(true)
try {
val res =
try {
screenRecorder.record(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
return GatewaySession.InvokeResult.error(code = code, message = message)
}
return GatewaySession.InvokeResult.ok(res.payloadJson)
} finally {
setScreenRecordActive(false)
}
}
}

View File

@@ -0,0 +1,19 @@
package ai.openclaw.android.node
import ai.openclaw.android.gateway.GatewaySession
class SmsHandler(
private val sms: SmsManager,
) {
suspend fun handleSmsSend(paramsJson: String?): GatewaySession.InvokeResult {
val res = sms.send(paramsJson)
if (res.ok) {
return GatewaySession.InvokeResult.ok(res.payloadJson)
} else {
val error = res.error ?: "SMS_SEND_FAILED"
val idx = error.indexOf(':')
val code = if (idx > 0) error.substring(0, idx).trim() else "SMS_SEND_FAILED"
return GatewaySession.InvokeResult.error(code = code, message = error)
}
}
}

View File

@@ -82,6 +82,7 @@ fun SettingsSheet(viewModel: MainViewModel) {
val manualHost by viewModel.manualHost.collectAsState()
val manualPort by viewModel.manualPort.collectAsState()
val manualTls by viewModel.manualTls.collectAsState()
val gatewayToken by viewModel.gatewayToken.collectAsState()
val canvasDebugStatusEnabled by viewModel.canvasDebugStatusEnabled.collectAsState()
val statusText by viewModel.statusText.collectAsState()
val serverName by viewModel.serverName.collectAsState()
@@ -403,6 +404,14 @@ fun SettingsSheet(viewModel: MainViewModel) {
modifier = Modifier.fillMaxWidth(),
enabled = manualEnabled,
)
OutlinedTextField(
value = gatewayToken,
onValueChange = viewModel::setGatewayToken,
label = { Text("Gateway Token") },
modifier = Modifier.fillMaxWidth(),
enabled = manualEnabled,
singleLine = true,
)
ListItem(
headlineContent = { Text("Require TLS") },
supportingContent = { Text("Pin the gateway certificate on first connect.") },

View File

@@ -37,6 +37,7 @@ import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import ai.openclaw.android.chat.ChatSessionEntry
@@ -63,8 +64,9 @@ fun ChatComposer(
var showSessionMenu by remember { mutableStateOf(false) }
val sessionOptions = resolveSessionChoices(sessionKey, sessions, mainSessionKey = mainSessionKey)
val currentSessionLabel =
val currentSessionLabel = friendlySessionName(
sessionOptions.firstOrNull { it.key == sessionKey }?.displayName ?: sessionKey
)
val canSend = pendingRunCount == 0 && (input.trim().isNotEmpty() || attachments.isNotEmpty()) && healthOk
@@ -76,7 +78,7 @@ fun ChatComposer(
) {
Column(modifier = Modifier.padding(10.dp), verticalArrangement = Arrangement.spacedBy(8.dp)) {
Row(
modifier = Modifier.fillMaxWidth(),
modifier = Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalAlignment = Alignment.CenterVertically,
) {
@@ -85,13 +87,13 @@ fun ChatComposer(
onClick = { showSessionMenu = true },
contentPadding = ButtonDefaults.ContentPadding,
) {
Text("Session: $currentSessionLabel")
Text(currentSessionLabel, maxLines = 1, overflow = TextOverflow.Ellipsis)
}
DropdownMenu(expanded = showSessionMenu, onDismissRequest = { showSessionMenu = false }) {
for (entry in sessionOptions) {
DropdownMenuItem(
text = { Text(entry.displayName ?: entry.key) },
text = { Text(friendlySessionName(entry.displayName ?: entry.key)) },
onClick = {
onSelectSession(entry.key)
showSessionMenu = false
@@ -113,7 +115,7 @@ fun ChatComposer(
onClick = { showThinkingMenu = true },
contentPadding = ButtonDefaults.ContentPadding,
) {
Text("Thinking: ${thinkingLabel(thinkingLevel)}")
Text("🧠 ${thinkingLabel(thinkingLevel)}", maxLines = 1)
}
DropdownMenu(expanded = showThinkingMenu, onDismissRequest = { showThinkingMenu = false }) {
@@ -124,8 +126,6 @@ fun ChatComposer(
}
}
Spacer(modifier = Modifier.weight(1f))
FilledTonalIconButton(onClick = onRefresh, modifier = Modifier.size(42.dp)) {
Icon(Icons.Default.Refresh, contentDescription = "Refresh")
}

View File

@@ -33,14 +33,9 @@ fun ChatMessageListCard(
) {
val listState = rememberLazyListState()
// With reverseLayout the newest item is at index 0 (bottom of screen).
LaunchedEffect(messages.size, pendingRunCount, pendingToolCalls.size, streamingAssistantText) {
val total =
messages.size +
(if (pendingRunCount > 0) 1 else 0) +
(if (pendingToolCalls.isNotEmpty()) 1 else 0) +
(if (!streamingAssistantText.isNullOrBlank()) 1 else 0)
if (total <= 0) return@LaunchedEffect
listState.animateScrollToItem(index = total - 1)
listState.animateScrollToItem(index = 0)
}
Card(
@@ -56,16 +51,17 @@ fun ChatMessageListCard(
LazyColumn(
modifier = Modifier.fillMaxSize(),
state = listState,
reverseLayout = true,
verticalArrangement = Arrangement.spacedBy(14.dp),
contentPadding = androidx.compose.foundation.layout.PaddingValues(top = 12.dp, bottom = 12.dp, start = 12.dp, end = 12.dp),
) {
items(count = messages.size, key = { idx -> messages[idx].id }) { idx ->
ChatMessageBubble(message = messages[idx])
}
// With reverseLayout = true, index 0 renders at the BOTTOM.
// So we emit newest items first: streaming → tools → typing → messages (newest→oldest).
if (pendingRunCount > 0) {
item(key = "typing") {
ChatTypingIndicatorBubble()
val stream = streamingAssistantText?.trim()
if (!stream.isNullOrEmpty()) {
item(key = "stream") {
ChatStreamingAssistantBubble(text = stream)
}
}
@@ -75,12 +71,15 @@ fun ChatMessageListCard(
}
}
val stream = streamingAssistantText?.trim()
if (!stream.isNullOrEmpty()) {
item(key = "stream") {
ChatStreamingAssistantBubble(text = stream)
if (pendingRunCount > 0) {
item(key = "typing") {
ChatTypingIndicatorBubble()
}
}
items(count = messages.size, key = { idx -> messages[messages.size - 1 - idx].id }) { idx ->
ChatMessageBubble(message = messages[messages.size - 1 - idx])
}
}
if (messages.isEmpty() && pendingRunCount == 0 && pendingToolCalls.isEmpty() && streamingAssistantText.isNullOrBlank()) {

View File

@@ -43,6 +43,17 @@ import androidx.compose.ui.platform.LocalContext
fun ChatMessageBubble(message: ChatMessage) {
val isUser = message.role.lowercase() == "user"
// Filter to only displayable content parts (text with content, or base64 images)
val displayableContent = message.content.filter { part ->
when (part.type) {
"text" -> !part.text.isNullOrBlank()
else -> part.base64 != null
}
}
// Skip rendering entirely if no displayable content
if (displayableContent.isEmpty()) return
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = if (isUser) Arrangement.End else Arrangement.Start,
@@ -61,7 +72,7 @@ fun ChatMessageBubble(message: ChatMessage) {
.padding(horizontal = 12.dp, vertical = 10.dp),
) {
val textColor = textColorOverBubble(isUser)
ChatMessageBody(content = message.content, textColor = textColor)
ChatMessageBody(content = displayableContent, textColor = textColor)
}
}
}

View File

@@ -4,6 +4,30 @@ import ai.openclaw.android.chat.ChatSessionEntry
private const val RECENT_WINDOW_MS = 24 * 60 * 60 * 1000L
/**
* Derive a human-friendly label from a raw session key.
* Examples:
* "telegram:g-agent-main-main" -> "Main"
* "agent:main:main" -> "Main"
* "discord:g-server-channel" -> "Server Channel"
* "my-custom-session" -> "My Custom Session"
*/
fun friendlySessionName(key: String): String {
// Strip common prefixes like "telegram:", "agent:", "discord:" etc.
val stripped = key.substringAfterLast(":")
// Remove leading "g-" prefix (gateway artifact)
val cleaned = if (stripped.startsWith("g-")) stripped.removePrefix("g-") else stripped
// Split on hyphens/underscores, title-case each word, collapse "main main" -> "Main"
val words = cleaned.split('-', '_').filter { it.isNotBlank() }.map { word ->
word.replaceFirstChar { it.uppercaseChar() }
}.distinct()
val result = words.joinToString(" ")
return result.ifBlank { key }
}
fun resolveSessionChoices(
currentSessionKey: String,
sessions: List<ChatSessionEntry>,

View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<paths>
<cache-path name="apk_updates" path="updates/" />
</paths>

View File

@@ -0,0 +1,65 @@
package ai.openclaw.android.node
import java.io.File
import org.junit.Assert.assertEquals
import org.junit.Assert.assertThrows
import org.junit.Test
class AppUpdateHandlerTest {
@Test
fun parseAppUpdateRequest_acceptsHttpsWithMatchingHost() {
val req =
parseAppUpdateRequest(
paramsJson =
"""{"url":"https://gw.example.com/releases/openclaw.apk","sha256":"${"a".repeat(64)}"}""",
connectedHost = "gw.example.com",
)
assertEquals("https://gw.example.com/releases/openclaw.apk", req.url)
assertEquals("a".repeat(64), req.expectedSha256)
}
@Test
fun parseAppUpdateRequest_rejectsNonHttps() {
assertThrows(IllegalArgumentException::class.java) {
parseAppUpdateRequest(
paramsJson = """{"url":"http://gw.example.com/releases/openclaw.apk","sha256":"${"a".repeat(64)}"}""",
connectedHost = "gw.example.com",
)
}
}
@Test
fun parseAppUpdateRequest_rejectsHostMismatch() {
assertThrows(IllegalArgumentException::class.java) {
parseAppUpdateRequest(
paramsJson = """{"url":"https://evil.example.com/releases/openclaw.apk","sha256":"${"a".repeat(64)}"}""",
connectedHost = "gw.example.com",
)
}
}
@Test
fun parseAppUpdateRequest_rejectsInvalidSha256() {
assertThrows(IllegalArgumentException::class.java) {
parseAppUpdateRequest(
paramsJson = """{"url":"https://gw.example.com/releases/openclaw.apk","sha256":"bad"}""",
connectedHost = "gw.example.com",
)
}
}
@Test
fun sha256Hex_computesExpectedDigest() {
val tmp = File.createTempFile("openclaw-update-hash", ".bin")
try {
tmp.writeText("hello", Charsets.UTF_8)
assertEquals(
"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
sha256Hex(tmp),
)
} finally {
tmp.delete()
}
}
}

View File

@@ -2,3 +2,4 @@ org.gradle.jvmargs=-Xmx3g -Dfile.encoding=UTF-8 --enable-native-access=ALL-UNNAM
org.gradle.warning.mode=none
android.useAndroidX=true
android.nonTransitiveRClass=true
android.enableR8.fullMode=true

View File

@@ -1,5 +1,5 @@
import type { ChildProcessWithoutNullStreams } from "node:child_process";
import { beforeEach, describe, expect, it } from "vitest";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { ProcessSession } from "./bash-process-registry.js";
import {
addSession,
@@ -20,7 +20,7 @@ describe("bash process registry", () => {
const session: ProcessSession = {
id: "sess",
command: "echo test",
child: { pid: 123 } as ChildProcessWithoutNullStreams,
child: { pid: 123, removeAllListeners: vi.fn() } as ChildProcessWithoutNullStreams,
startedAt: Date.now(),
cwd: "/tmp",
maxOutputChars: 10,
@@ -51,7 +51,7 @@ describe("bash process registry", () => {
const session: ProcessSession = {
id: "sess",
command: "echo test",
child: { pid: 123 } as ChildProcessWithoutNullStreams,
child: { pid: 123, removeAllListeners: vi.fn() } as ChildProcessWithoutNullStreams,
startedAt: Date.now(),
cwd: "/tmp",
maxOutputChars: 100_000,
@@ -85,7 +85,7 @@ describe("bash process registry", () => {
const session: ProcessSession = {
id: "sess",
command: "echo test",
child: { pid: 123 } as ChildProcessWithoutNullStreams,
child: { pid: 123, removeAllListeners: vi.fn() } as ChildProcessWithoutNullStreams,
startedAt: Date.now(),
cwd: "/tmp",
maxOutputChars: 5_000,
@@ -116,7 +116,7 @@ describe("bash process registry", () => {
const session: ProcessSession = {
id: "sess",
command: "echo test",
child: { pid: 123 } as ChildProcessWithoutNullStreams,
child: { pid: 123, removeAllListeners: vi.fn() } as ChildProcessWithoutNullStreams,
startedAt: Date.now(),
cwd: "/tmp",
maxOutputChars: 100,
@@ -150,7 +150,7 @@ describe("bash process registry", () => {
const session: ProcessSession = {
id: "sess",
command: "echo test",
child: { pid: 123 } as ChildProcessWithoutNullStreams,
child: { pid: 123, removeAllListeners: vi.fn() } as ChildProcessWithoutNullStreams,
startedAt: Date.now(),
cwd: "/tmp",
maxOutputChars: 100,

View File

@@ -168,7 +168,7 @@ function moveToFinished(session: ProcessSession, status: ProcessStatus) {
session.child.stderr?.destroy?.();
// Remove all event listeners to prevent memory leaks
session.child.removeAllListeners?.();
session.child.removeAllListeners();
// Clear the reference
delete session.child;

View File

@@ -8,6 +8,7 @@ import {
parseCameraClipPayload,
parseCameraSnapPayload,
writeBase64ToFile,
writeUrlToFile,
} from "../../cli/nodes-camera.js";
import { parseEnvPairs, parseTimeoutMs } from "../../cli/nodes-run.js";
import {
@@ -230,14 +231,20 @@ export function createNodesTool(options?: {
facing,
ext: isJpeg ? "jpg" : "png",
});
await writeBase64ToFile(filePath, payload.base64);
if (payload.url) {
await writeUrlToFile(filePath, payload.url);
} else if (payload.base64) {
await writeBase64ToFile(filePath, payload.base64);
}
content.push({ type: "text", text: `MEDIA:${filePath}` });
content.push({
type: "image",
data: payload.base64,
mimeType:
imageMimeFromFormat(payload.format) ?? (isJpeg ? "image/jpeg" : "image/png"),
});
if (payload.base64) {
content.push({
type: "image",
data: payload.base64,
mimeType:
imageMimeFromFormat(payload.format) ?? (isJpeg ? "image/jpeg" : "image/png"),
});
}
details.push({
facing,
path: filePath,
@@ -300,7 +307,11 @@ export function createNodesTool(options?: {
facing,
ext: payload.format,
});
await writeBase64ToFile(filePath, payload.base64);
if (payload.url) {
await writeUrlToFile(filePath, payload.url);
} else if (payload.base64) {
await writeBase64ToFile(filePath, payload.base64);
}
return {
content: [{ type: "text", text: `FILE:${filePath}` }],
details: {

View File

@@ -1,12 +1,13 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import { describe, expect, it } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
cameraTempPath,
parseCameraClipPayload,
parseCameraSnapPayload,
writeBase64ToFile,
writeUrlToFile,
} from "./nodes-camera.js";
describe("nodes camera helpers", () => {
@@ -61,4 +62,45 @@ describe("nodes camera helpers", () => {
await expect(fs.readFile(out, "utf8")).resolves.toBe("hi");
await fs.rm(dir, { recursive: true, force: true });
});
afterEach(() => {
vi.unstubAllGlobals();
});
it("writes url payload to file", async () => {
vi.stubGlobal(
"fetch",
vi.fn(async () => new Response("url-content", { status: 200 })),
);
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-test-"));
const out = path.join(dir, "x.bin");
try {
await writeUrlToFile(out, "https://example.com/clip.mp4");
await expect(fs.readFile(out, "utf8")).resolves.toBe("url-content");
} finally {
await fs.rm(dir, { recursive: true, force: true });
}
});
it("rejects non-https url payload", async () => {
await expect(writeUrlToFile("/tmp/ignored", "http://example.com/x.bin")).rejects.toThrow(
/only https/i,
);
});
it("rejects oversized content-length for url payload", async () => {
vi.stubGlobal(
"fetch",
vi.fn(
async () =>
new Response("tiny", {
status: 200,
headers: { "content-length": String(999_999_999) },
}),
),
);
await expect(writeUrlToFile("/tmp/ignored", "https://example.com/huge.bin")).rejects.toThrow(
/exceeds max/i,
);
});
});

View File

@@ -4,18 +4,22 @@ import * as os from "node:os";
import * as path from "node:path";
import { resolveCliName } from "./cli-name.js";
const MAX_CAMERA_URL_DOWNLOAD_BYTES = 250 * 1024 * 1024;
export type CameraFacing = "front" | "back";
export type CameraSnapPayload = {
format: string;
base64: string;
base64?: string;
url?: string;
width: number;
height: number;
};
export type CameraClipPayload = {
format: string;
base64: string;
base64?: string;
url?: string;
durationMs: number;
hasAudio: boolean;
};
@@ -40,24 +44,26 @@ export function parseCameraSnapPayload(value: unknown): CameraSnapPayload {
const obj = asRecord(value);
const format = asString(obj.format);
const base64 = asString(obj.base64);
const url = asString(obj.url);
const width = asNumber(obj.width);
const height = asNumber(obj.height);
if (!format || !base64 || width === undefined || height === undefined) {
if (!format || (!base64 && !url) || width === undefined || height === undefined) {
throw new Error("invalid camera.snap payload");
}
return { format, base64, width, height };
return { format, ...(base64 ? { base64 } : {}), ...(url ? { url } : {}), width, height };
}
export function parseCameraClipPayload(value: unknown): CameraClipPayload {
const obj = asRecord(value);
const format = asString(obj.format);
const base64 = asString(obj.base64);
const url = asString(obj.url);
const durationMs = asNumber(obj.durationMs);
const hasAudio = asBoolean(obj.hasAudio);
if (!format || !base64 || durationMs === undefined || hasAudio === undefined) {
if (!format || (!base64 && !url) || durationMs === undefined || hasAudio === undefined) {
throw new Error("invalid camera.clip payload");
}
return { format, base64, durationMs, hasAudio };
return { format, ...(base64 ? { base64 } : {}), ...(url ? { url } : {}), durationMs, hasAudio };
}
export function cameraTempPath(opts: {
@@ -75,6 +81,69 @@ export function cameraTempPath(opts: {
return path.join(tmpDir, `${cliName}-camera-${opts.kind}${facingPart}-${id}${ext}`);
}
export async function writeUrlToFile(filePath: string, url: string) {
const parsed = new URL(url);
if (parsed.protocol !== "https:") {
throw new Error(`writeUrlToFile: only https URLs are allowed, got ${parsed.protocol}`);
}
const res = await fetch(url);
if (!res.ok) {
throw new Error(`failed to download ${url}: ${res.status} ${res.statusText}`);
}
const contentLengthRaw = res.headers.get("content-length");
const contentLength = contentLengthRaw ? Number.parseInt(contentLengthRaw, 10) : undefined;
if (
typeof contentLength === "number" &&
Number.isFinite(contentLength) &&
contentLength > MAX_CAMERA_URL_DOWNLOAD_BYTES
) {
throw new Error(
`writeUrlToFile: content-length ${contentLength} exceeds max ${MAX_CAMERA_URL_DOWNLOAD_BYTES}`,
);
}
const body = res.body;
if (!body) {
throw new Error(`failed to download ${url}: empty response body`);
}
const fileHandle = await fs.open(filePath, "w");
let bytes = 0;
let thrown: unknown;
try {
const reader = body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
if (!value || value.byteLength === 0) {
continue;
}
bytes += value.byteLength;
if (bytes > MAX_CAMERA_URL_DOWNLOAD_BYTES) {
throw new Error(
`writeUrlToFile: downloaded ${bytes} bytes, exceeds max ${MAX_CAMERA_URL_DOWNLOAD_BYTES}`,
);
}
await fileHandle.write(value);
}
} catch (err) {
thrown = err;
} finally {
await fileHandle.close();
}
if (thrown) {
await fs.unlink(filePath).catch(() => {});
throw thrown;
}
return { path: filePath, bytes };
}
export async function writeBase64ToFile(filePath: string, base64: string) {
const buf = Buffer.from(base64, "base64");
await fs.writeFile(filePath, buf);

View File

@@ -10,6 +10,7 @@ import {
parseCameraClipPayload,
parseCameraSnapPayload,
writeBase64ToFile,
writeUrlToFile,
} from "../nodes-camera.js";
import { parseDurationMs } from "../parse-duration.js";
import { getNodesTheme, runNodesCommand } from "./cli-utils.js";
@@ -155,7 +156,11 @@ export function registerNodesCameraCommands(nodes: Command) {
facing,
ext: payload.format === "jpeg" ? "jpg" : payload.format,
});
await writeBase64ToFile(filePath, payload.base64);
if (payload.url) {
await writeUrlToFile(filePath, payload.url);
} else if (payload.base64) {
await writeBase64ToFile(filePath, payload.base64);
}
results.push({
facing,
path: filePath,
@@ -223,7 +228,11 @@ export function registerNodesCameraCommands(nodes: Command) {
facing,
ext: payload.format,
});
await writeBase64ToFile(filePath, payload.base64);
if (payload.url) {
await writeUrlToFile(filePath, payload.url);
} else if (payload.base64) {
await writeBase64ToFile(filePath, payload.base64);
}
if (opts.json) {
defaultRuntime.log(

View File

@@ -1,5 +1,6 @@
import * as fs from "node:fs/promises";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { parseCameraSnapPayload, parseCameraClipPayload } from "./nodes-camera.js";
const messageCommand = vi.fn();
const statusCommand = vi.fn();
@@ -461,4 +462,171 @@ describe("cli program (nodes media)", () => {
true,
);
});
describe("URL-based payloads", () => {
let originalFetch: typeof globalThis.fetch;
beforeAll(() => {
originalFetch = globalThis.fetch;
globalThis.fetch = vi.fn(
async () =>
new Response("url-content", {
status: 200,
headers: { "content-length": String("11") },
}),
) as unknown as typeof globalThis.fetch;
});
afterAll(() => {
globalThis.fetch = originalFetch;
});
it("runs nodes camera snap with url payload", async () => {
callGateway.mockImplementation(async (opts: { method?: string }) => {
if (opts.method === "node.list") {
return {
ts: Date.now(),
nodes: [
{
nodeId: "ios-node",
displayName: "iOS Node",
remoteIp: "192.168.0.88",
connected: true,
},
],
};
}
if (opts.method === "node.invoke") {
return {
ok: true,
nodeId: "ios-node",
command: "camera.snap",
payload: {
format: "jpg",
url: "https://example.com/photo.jpg",
width: 640,
height: 480,
},
};
}
return { ok: true };
});
const program = buildProgram();
runtime.log.mockClear();
await program.parseAsync(
["nodes", "camera", "snap", "--node", "ios-node", "--facing", "front"],
{ from: "user" },
);
const out = String(runtime.log.mock.calls[0]?.[0] ?? "");
const mediaPath = out.replace(/^MEDIA:/, "").trim();
expect(mediaPath).toMatch(/openclaw-camera-snap-front-.*\.jpg$/);
try {
await expect(fs.readFile(mediaPath, "utf8")).resolves.toBe("url-content");
} finally {
await fs.unlink(mediaPath).catch(() => {});
}
});
it("runs nodes camera clip with url payload", async () => {
callGateway.mockImplementation(async (opts: { method?: string }) => {
if (opts.method === "node.list") {
return {
ts: Date.now(),
nodes: [
{
nodeId: "ios-node",
displayName: "iOS Node",
remoteIp: "192.168.0.88",
connected: true,
},
],
};
}
if (opts.method === "node.invoke") {
return {
ok: true,
nodeId: "ios-node",
command: "camera.clip",
payload: {
format: "mp4",
url: "https://example.com/clip.mp4",
durationMs: 5000,
hasAudio: true,
},
};
}
return { ok: true };
});
const program = buildProgram();
runtime.log.mockClear();
await program.parseAsync(
["nodes", "camera", "clip", "--node", "ios-node", "--duration", "5000"],
{ from: "user" },
);
const out = String(runtime.log.mock.calls[0]?.[0] ?? "");
const mediaPath = out.replace(/^MEDIA:/, "").trim();
expect(mediaPath).toMatch(/openclaw-camera-clip-front-.*\.mp4$/);
try {
await expect(fs.readFile(mediaPath, "utf8")).resolves.toBe("url-content");
} finally {
await fs.unlink(mediaPath).catch(() => {});
}
});
});
describe("parseCameraSnapPayload with url", () => {
it("accepts url without base64", () => {
const result = parseCameraSnapPayload({
format: "jpg",
url: "https://example.com/photo.jpg",
width: 640,
height: 480,
});
expect(result.url).toBe("https://example.com/photo.jpg");
expect(result.base64).toBeUndefined();
});
it("accepts both base64 and url", () => {
const result = parseCameraSnapPayload({
format: "jpg",
base64: "aGk=",
url: "https://example.com/photo.jpg",
width: 640,
height: 480,
});
expect(result.base64).toBe("aGk=");
expect(result.url).toBe("https://example.com/photo.jpg");
});
it("rejects payload with neither base64 nor url", () => {
expect(() => parseCameraSnapPayload({ format: "jpg", width: 640, height: 480 })).toThrow(
"invalid camera.snap payload",
);
});
});
describe("parseCameraClipPayload with url", () => {
it("accepts url without base64", () => {
const result = parseCameraClipPayload({
format: "mp4",
url: "https://example.com/clip.mp4",
durationMs: 3000,
hasAudio: true,
});
expect(result.url).toBe("https://example.com/clip.mp4");
expect(result.base64).toBeUndefined();
});
it("rejects payload with neither base64 nor url", () => {
expect(() =>
parseCameraClipPayload({ format: "mp4", durationMs: 3000, hasAudio: true }),
).toThrow("invalid camera.clip payload");
});
});
});