feat: add copyVideoFrameAt and saveVideoFrameAs methods on webContents (#48149)

* feat: add copyVideoFrameAt and saveVideoFrameAs Method on Webcontent

chore: change the description of savevideoframe api

chore: add the description of the restrictive elements for using the APIs.

move to webframemain

fixed mediaPlayerAction to kSaveVideoFrameAs

Update spec/api-web-frame-main-spec.ts

Co-authored-by: John Kleinschmidt <kleinschmidtorama@gmail.com>

Update spec/api-web-frame-main-spec.ts

Co-authored-by: John Kleinschmidt <kleinschmidtorama@gmail.com>

fixed clipboard tests for video frame copying

fixed test for copying video frame to clipboard. check video loaded before copy video frame in test.

chore: try non-proprietary video format

Revert "chore: try non-proprietary video format"

This reverts commit ef085f88a1af53b6408a7af695cc60b8681398cf.

fix: format video as file url

* test: skip webFrameMain.copyVideoFrameAt on win32 CI due Chromium DCHECK

docs/api/web-contents.md

@@ -1585,6 +1585,20 @@ Centers the current text selection in web page.
 
 Copy the image at the given position to the clipboard.
 
+#### `contents.copyVideoFrameAt(x, y)`
+
+* `x` Integer
+* `y` Integer
+
+When executed on a video media element, copies the frame at (x, y) to the clipboard.
+
+#### `contents.saveVideoFrameAs(x, y)`
+
+* `x` Integer
+* `y` Integer
+
+When executed on a video media element, shows a save dialog and saves the frame at (x, y) to disk.
+
 #### `contents.paste()`
 
 Executes the editing command `paste` in web page.

docs/api/web-frame-main.md

@@ -175,6 +175,20 @@ app.on('web-contents-created', (_, webContents) => {
 })
 ```
 
+#### `frame.copyVideoFrameAt(x, y)`
+
+* `x` Integer
+* `y` Integer
+
+When executed on a video media element, copies the frame at (x, y) to the clipboard.
+
+#### `frame.saveVideoFrameAs(x, y)`
+
+* `x` Integer
+* `y` Integer
+
+When executed on a video media element, shows a save dialog and saves the frame at (x, y) to disk.
+
 ### Instance Properties
 
 #### `frame.ipc` _Readonly_

lib/browser/api/web-contents.ts

@@ -437,6 +437,14 @@ WebContents.prototype.loadURL = function (url, options) {
   return p;
 };
 
+WebContents.prototype.copyVideoFrameAt = function (x: number, y: number) {
+  this.mainFrame.copyVideoFrameAt(x, y);
+};
+
+WebContents.prototype.saveVideoFrameAs = function (x: number, y: number) {
+  this.mainFrame.saveVideoFrameAs(x, y);
+};
+
 WebContents.prototype.setWindowOpenHandler = function (handler: (details: Electron.HandlerDetails) => Electron.WindowOpenHandlerResponse) {
   this._windowOpenHandler = handler;
 };

shell/browser/api/electron_api_web_frame_main.cc

@@ -36,6 +36,8 @@
 #include "shell/common/node_includes.h"
 #include "shell/common/v8_util.h"
 #include "third_party/abseil-cpp/absl/container/flat_hash_map.h"
+#include "third_party/blink/public/mojom/frame/media_player_action.mojom.h"
+#include "ui/gfx/geometry/point.h"
 
 namespace {
 
@@ -260,6 +262,28 @@ v8::Local<v8::Promise> WebFrameMain::ExecuteJavaScript(
   return handle;
 }
 
+void WebFrameMain::CopyVideoFrameAt(int x, int y) {
+  if (!CheckRenderFrame())
+    return;
+  auto location = gfx::Point(x, y);
+  auto action = blink::mojom::MediaPlayerAction(
+      blink::mojom::MediaPlayerActionType::kCopyVideoFrame,
+      /*enable=*/true);
+  return render_frame_host()->ExecuteMediaPlayerActionAtLocation(location,
+                                                                 action);
+}
+
+void WebFrameMain::SaveVideoFrameAs(int x, int y) {
+  if (!CheckRenderFrame())
+    return;
+  auto location = gfx::Point(x, y);
+  auto action = blink::mojom::MediaPlayerAction(
+      blink::mojom::MediaPlayerActionType::kSaveVideoFrameAs,
+      /*enable=*/true);
+  return render_frame_host()->ExecuteMediaPlayerActionAtLocation(location,
+                                                                 action);
+}
+
 bool WebFrameMain::Reload() {
   if (!CheckRenderFrame())
     return false;
@@ -593,6 +617,8 @@ void WebFrameMain::FillObjectTemplate(v8::Isolate* isolate,
       .SetMethod("executeJavaScript", &WebFrameMain::ExecuteJavaScript)
       .SetMethod("collectJavaScriptCallStack",
                  &WebFrameMain::CollectDocumentJSCallStack)
+      .SetMethod("copyVideoFrameAt", &WebFrameMain::CopyVideoFrameAt)
+      .SetMethod("saveVideoFrameAs", &WebFrameMain::SaveVideoFrameAs)
       .SetMethod("reload", &WebFrameMain::Reload)
       .SetMethod("isDestroyed", &WebFrameMain::IsDestroyed)
       .SetMethod("_send", &WebFrameMain::Send)

shell/browser/api/electron_api_web_frame_main.h

@@ -118,6 +118,8 @@ class WebFrameMain final : public gin_helper::DeprecatedWrappable<WebFrameMain>,
 
   v8::Local<v8::Promise> ExecuteJavaScript(gin::Arguments* args,
                                            const std::u16string& code);
+  void CopyVideoFrameAt(int x, int y);
+  void SaveVideoFrameAs(int x, int y);
   bool Reload();
   bool IsDestroyed() const;
   void Send(v8::Isolate* isolate,

spec/api-web-frame-main-spec.ts

@@ -1,3 +1,4 @@
+import { clipboard } from 'electron/common';
 import { BrowserWindow, WebFrameMain, webFrameMain, ipcMain, app, WebContents } from 'electron/main';
 
 import { expect } from 'chai';
@@ -534,6 +535,91 @@ describe('webFrameMain module', () => {
     });
   });
 
+  describe('webFrameMain.copyVideoFrameAt', () => {
+    const insertVideoInFrame = async (frame: WebFrameMain) => {
+      const videoFilePath = url.pathToFileURL(path.join(fixtures, 'cat-spin.mp4')).href;
+      await frame.executeJavaScript(`
+        const video = document.createElement('video');
+        video.src = '${videoFilePath}';
+        video.muted = true;
+        video.loop = true;
+        video.play();
+        document.body.appendChild(video);
+      `);
+    };
+
+    const getFramePosition = async (frame: WebFrameMain) => {
+      const point = await frame.executeJavaScript(`(${() => {
+        const iframe = document.querySelector('iframe');
+        if (!iframe) return;
+        const rect = iframe.getBoundingClientRect();
+        return { x: Math.floor(rect.x), y: Math.floor(rect.y) };
+      }})()`) as Electron.Point;
+      expect(point).to.be.an('object');
+      return point;
+    };
+
+    const copyVideoFrameInFrame = async (frame: WebFrameMain) => {
+      const point = await frame.executeJavaScript(`(${() => {
+        const video = document.querySelector('video');
+        if (!video) return;
+        const rect = video.getBoundingClientRect();
+        return {
+          x: Math.floor(rect.x + rect.width / 2),
+          y: Math.floor(rect.y + rect.height / 2)
+        };
+      }})()`) as Electron.Point;
+
+      expect(point).to.be.an('object');
+
+      // Translate coordinate to be relative of main frame
+      if (frame.parent) {
+        const framePosition = await getFramePosition(frame.parent);
+        point.x += framePosition.x;
+        point.y += framePosition.y;
+      }
+
+      expect(clipboard.readImage().isEmpty()).to.be.true();
+      // wait for video to load
+      await frame.executeJavaScript(`(${() => {
+        const video = document.querySelector('video');
+        if (!video) return;
+        return new Promise(resolve => {
+          if (video.readyState >= 4) resolve(null);
+          else video.addEventListener('canplaythrough', resolve, { once: true });
+        });
+      }})()`);
+      frame.copyVideoFrameAt(point.x, point.y);
+      await waitUntil(() => clipboard.availableFormats().includes('image/png'));
+      expect(clipboard.readImage().isEmpty()).to.be.false();
+    };
+
+    beforeEach(() => {
+      clipboard.clear();
+    });
+
+    // TODO: Re-enable on Windows CI once Chromium fixes the intermittent
+    // backwards-time DCHECK hit while copying video frames:
+    // DCHECK failed: !delta.is_negative().
+    ifit(!(process.platform === 'win32' && process.env.CI))('copies video frame in main frame', async () => {
+      const w = new BrowserWindow({ show: false });
+      await w.webContents.loadFile(path.join(fixtures, 'blank.html'));
+      await insertVideoInFrame(w.webContents.mainFrame);
+      await copyVideoFrameInFrame(w.webContents.mainFrame);
+      await waitUntil(() => clipboard.availableFormats().includes('image/png'));
+    });
+
+    ifit(!(process.platform === 'win32' && process.env.CI))('copies video frame in subframe', async () => {
+      const w = new BrowserWindow({ show: false });
+      await w.webContents.loadFile(path.join(subframesPath, 'frame-with-frame.html'));
+      const subframe = w.webContents.mainFrame.frames[0];
+      expect(subframe).to.exist();
+      await insertVideoInFrame(subframe);
+      await copyVideoFrameInFrame(subframe);
+      await waitUntil(() => clipboard.availableFormats().includes('image/png'));
+    });
+  });
+
   describe('"frame-created" event', () => {
     it('emits when the main frame is created', async () => {
       const w = new BrowserWindow({ show: false });