fix: ensure nativeImage serialization main->renderer (#23794)

* refactor: use typeutils for nativeImage serialization (#23693)

* fix: ensure nativeImage serialization main->renderer

lib/browser/remote/server.ts

@@ -4,12 +4,13 @@ import * as electron from 'electron';
 import { EventEmitter } from 'events';
 import objectsRegistry from './objects-registry';
 import { ipcMainInternal } from '../ipc-main-internal';
-import { isPromise, isSerializableObject } from '@electron/internal/common/type-utils';
+import { isPromise, isSerializableObject, deserialize, serialize } from '@electron/internal/common/type-utils';
 import { Size } from 'electron/main';
 
 const v8Util = process.electronBinding('v8_util');
 const eventBinding = process.electronBinding('event');
 const features = process.electronBinding('features');
+const { NativeImage } = process.electronBinding('native_image');
 
 if (!features.isRemoteModuleEnabled()) {
   throw new Error('remote module is disabled');
@@ -114,6 +115,9 @@ type MetaType = {
 } | {
   type: 'promise',
   then: MetaType
+} | {
+  type: 'nativeimage'
+  value: electron.NativeImage
 }
 
 // Convert a real value into meta data.
@@ -124,6 +128,8 @@ const valueToMeta = function (sender: electron.WebContents, contextId: string, v
     // Recognize certain types of objects.
     if (value instanceof Buffer) {
       type = 'buffer';
+    } else if (value instanceof NativeImage) {
+      type = 'nativeimage';
     } else if (Array.isArray(value)) {
       type = 'array';
     } else if (value instanceof Error) {
@@ -147,6 +153,8 @@ const valueToMeta = function (sender: electron.WebContents, contextId: string, v
       type,
       members: value.map((el: any) => valueToMeta(sender, contextId, el, optimizeSimpleObject))
     };
+  } else if (type === 'nativeimage') {
+    return { type, value: serialize(value) };
   } else if (type === 'object' || type === 'function') {
     return {
       type,
@@ -234,7 +242,10 @@ type MetaTypeFromRenderer = {
 } | {
   type: 'object',
   name: string,
-  members: { name: string, value: MetaTypeFromRenderer }[]
+  members: {
+    name: string,
+    value: MetaTypeFromRenderer
+  }[]
 } | {
   type: 'function-with-return-value',
   value: MetaTypeFromRenderer
@@ -245,7 +256,12 @@ type MetaTypeFromRenderer = {
   length: number
 } | {
   type: 'nativeimage',
-  value: { size: Size, buffer: Buffer, scaleFactor: number, dataURL: string }[]
+  value: {
+    size: Size,
+    buffer: Buffer,
+    scaleFactor: number,
+    dataURL: string
+  }[]
 }
 
 const fakeConstructor = (constructor: Function, name: string) =>
@@ -263,15 +279,8 @@ const fakeConstructor = (constructor: Function, name: string) =>
 const unwrapArgs = function (sender: electron.WebContents, frameId: number, contextId: string, args: any[]) {
   const metaToValue = function (meta: MetaTypeFromRenderer): any {
     switch (meta.type) {
-      case 'nativeimage': {
-        const image = electron.nativeImage.createEmpty();
-        for (const rep of meta.value) {
-          const { size, scaleFactor, dataURL } = rep;
-          const { width, height } = size;
-          image.addRepresentation({ dataURL, scaleFactor, width, height });
-        }
-        return image;
-      }
+      case 'nativeimage':
+        return deserialize(meta.value);
       case 'value':
         return meta.value;
       case 'remote-object':

lib/common/type-utils.ts

@@ -20,10 +20,10 @@ const serializableTypes = [
   Date,
   Error,
   RegExp,
-  ArrayBuffer,
-  NativeImage
+  ArrayBuffer
 ];
 
+// https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm#Supported_types
 export function isSerializableObject (value: any) {
   return value === null || ArrayBuffer.isView(value) || serializableTypes.some(type => value instanceof type);
 }
@@ -34,18 +34,55 @@ const objectMap = function (source: Object, mapper: (value: any) => any) {
   return Object.fromEntries(targetEntries);
 };
 
-export function serialize (value: any): any {
-  if (value instanceof NativeImage) {
-    const representations = [];
-    for (const scaleFactor of value.getScaleFactors()) {
-      const size = value.getSize(scaleFactor);
-      const dataURL = value.toDataURL({ scaleFactor });
+function serializeNativeImage (image: any) {
+  const representations = [];
+  const scaleFactors = image.getScaleFactors();
+
+  // Use Buffer when there's only one representation for better perf.
+  // This avoids compressing to/from PNG where it's not necessary to
+  // ensure uniqueness of dataURLs (since there's only one).
+  if (scaleFactors.length === 1) {
+    const scaleFactor = scaleFactors[0];
+    const size = image.getSize(scaleFactor);
+    const buffer = image.toBitmap({ scaleFactor });
+    representations.push({ scaleFactor, size, buffer });
+  } else {
+    // Construct from dataURLs to ensure that they are not lost in creation.
+    for (const scaleFactor of scaleFactors) {
+      const size = image.getSize(scaleFactor);
+      const dataURL = image.toDataURL({ scaleFactor });
       representations.push({ scaleFactor, size, dataURL });
     }
-    return { __ELECTRON_SERIALIZED_NativeImage__: true, representations };
-  } else if (value instanceof Buffer) {
-    return { __ELECTRON_SERIALIZED_Buffer__: true, data: value };
-  } else if (Array.isArray(value)) {
+  }
+  return { __ELECTRON_SERIALIZED_NativeImage__: true, representations };
+}
+
+function deserializeNativeImage (value: any) {
+  const image = nativeImage.createEmpty();
+
+  // Use Buffer when there's only one representation for better perf.
+  // This avoids compressing to/from PNG where it's not necessary to
+  // ensure uniqueness of dataURLs (since there's only one).
+  if (value.representations.length === 1) {
+    const { buffer, size, scaleFactor } = value.representations[0];
+    const { width, height } = size;
+    image.addRepresentation({ buffer, scaleFactor, width, height });
+  } else {
+    // Construct from dataURLs to ensure that they are not lost in creation.
+    for (const rep of value.representations) {
+      const { dataURL, size, scaleFactor } = rep;
+      const { width, height } = size;
+      image.addRepresentation({ dataURL, scaleFactor, width, height });
+    }
+  }
+
+  return image;
+}
+
+export function serialize (value: any): any {
+  if (value instanceof NativeImage) {
+    return serializeNativeImage(value);
+  } if (Array.isArray(value)) {
     return value.map(serialize);
   } else if (isSerializableObject(value)) {
     return value;
@@ -58,16 +95,7 @@ export function serialize (value: any): any {
 
 export function deserialize (value: any): any {
   if (value && value.__ELECTRON_SERIALIZED_NativeImage__) {
-    const image = nativeImage.createEmpty();
-    for (const rep of value.representations) {
-      const { size, scaleFactor, dataURL } = rep;
-      const { width, height } = size;
-      image.addRepresentation({ dataURL, scaleFactor, width, height });
-    }
-    return image;
-  } else if (value && value.__ELECTRON_SERIALIZED_Buffer__) {
-    const { buffer, byteOffset, byteLength } = value.data;
-    return Buffer.from(buffer, byteOffset, byteLength);
+    return deserializeNativeImage(value);
   } else if (Array.isArray(value)) {
     return value.map(deserialize);
   } else if (isSerializableObject(value)) {

lib/renderer/api/remote.js

@@ -5,7 +5,7 @@ const { hasSwitch } = process.electronBinding('command_line');
 const { NativeImage } = process.electronBinding('native_image');
 
 const { CallbacksRegistry } = require('@electron/internal/renderer/remote/callbacks-registry');
-const { isPromise, isSerializableObject } = require('@electron/internal/common/type-utils');
+const { isPromise, isSerializableObject, serialize, deserialize } = require('@electron/internal/common/type-utils');
 const { ipcRendererInternal } = require('@electron/internal/renderer/ipc-renderer-internal');
 
 const callbacksRegistry = new CallbacksRegistry();
@@ -37,14 +37,7 @@ function wrapArgs (args, visited = new Set()) {
     }
 
     if (value instanceof NativeImage) {
-      const images = [];
-      for (const scaleFactor of value.getScaleFactors()) {
-        const size = value.getSize(scaleFactor);
-        const buffer = value.toBitmap({ scaleFactor });
-        const dataURL = value.toDataURL({ scaleFactor });
-        images.push({ buffer, scaleFactor, size, dataURL });
-      }
-      return { type: 'nativeimage', value: images };
+      return { type: 'nativeimage', value: serialize(value) };
     } else if (Array.isArray(value)) {
       visited.add(value);
       const meta = {
@@ -226,6 +219,7 @@ function metaToValue (meta) {
   const types = {
     value: () => meta.value,
     array: () => meta.members.map((member) => metaToValue(member)),
+    nativeimage: () => deserialize(meta.value),
     buffer: () => Buffer.from(meta.value.buffer, meta.value.byteOffset, meta.value.byteLength),
     promise: () => Promise.resolve({ then: metaToValue(meta.then) }),
     error: () => metaToError(meta),

spec-main/api-remote-spec.ts

@@ -366,11 +366,11 @@ ifdescribe(features.isRemoteModuleEnabled())('remote module', () => {
     const w = makeWindow();
     const remotely = makeRemotely(w);
 
-    it('can serialize an empty nativeImage', async () => {
-      const getEmptyImage = (img: NativeImage) => img.isEmpty();
+    it('can serialize an empty nativeImage from renderer to main', async () => {
+      const getImageEmpty = (img: NativeImage) => img.isEmpty();
 
       w().webContents.once('remote-get-global', (event) => {
-        event.returnValue = getEmptyImage;
+        event.returnValue = getImageEmpty;
       });
 
       await expect(remotely(() => {
@@ -379,11 +379,23 @@ ifdescribe(features.isRemoteModuleEnabled())('remote module', () => {
       })).to.eventually.be.true();
     });
 
-    it('can serialize a non-empty nativeImage', async () => {
-      const getNonEmptyImage = (img: NativeImage) => img.getSize();
+    it('can serialize an empty nativeImage from main to renderer', async () => {
+      w().webContents.once('remote-get-global', (event) => {
+        const emptyImage = require('electron').nativeImage.createEmpty();
+        event.returnValue = emptyImage;
+      });
+
+      await expect(remotely(() => {
+        const image = require('electron').remote.getGlobal('someFunction');
+        return image.isEmpty();
+      })).to.eventually.be.true();
+    });
+
+    it('can serialize a non-empty nativeImage from renderer to main', async () => {
+      const getImageSize = (img: NativeImage) => img.getSize();
 
       w().webContents.once('remote-get-global', (event) => {
-        event.returnValue = getNonEmptyImage;
+        event.returnValue = getImageSize;
       });
 
       await expect(remotely(() => {
@@ -393,6 +405,18 @@ ifdescribe(features.isRemoteModuleEnabled())('remote module', () => {
       })).to.eventually.deep.equal({ width: 2, height: 2 });
     });
 
+    it('can serialize a non-empty nativeImage from main to renderer', async () => {
+      w().webContents.once('remote-get-global', (event) => {
+        const nonEmptyImage = nativeImage.createFromDataURL('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAAFklEQVQYlWP8//8/AwMDEwMDAwMDAwAkBgMBBMzldwAAAABJRU5ErkJggg==');
+        event.returnValue = nonEmptyImage;
+      });
+
+      await expect(remotely(() => {
+        const image = require('electron').remote.getGlobal('someFunction');
+        return image.getSize();
+      })).to.eventually.deep.equal({ width: 2, height: 2 });
+    });
+
     it('can properly create a menu with an nativeImage icon in the renderer', async () => {
       await expect(remotely(() => {
         const { remote, nativeImage } = require('electron');