feat: allow screen or window to return depending on preferredDisplaySurface set

docs/api/desktop-capturer.md

@@ -17,7 +17,7 @@ app.whenReady().then(() => {
 
   session.defaultSession.setDisplayMediaRequestHandler((request, callback) => {
     desktopCapturer.getSources({ types: ['screen'] }).then((sources) => {
-      // Your app shows some UI, but in this exampe
+      // Your app shows some UI, but in this example
       // grant access to the first screen found.
       callback({ video: sources[0], audio: 'loopback' })
     })

docs/api/session.md

@@ -971,6 +971,7 @@ session.fromPartition('some-partition').setPermissionCheckHandler((webContents,
     * `videoRequested` Boolean - true if the web content requested a video stream.
     * `audioRequested` Boolean - true if the web content requested an audio stream.
     * `userGesture` Boolean - Whether a user gesture was active when this request was triggered.
+    * `preferredDisplaySurface` String - The preferred display used for sharing screen in this request.
   * `callback` Function
     * `streams` Object
       * `video` Object | [WebFrameMain](web-frame-main.md) (optional)

filenames.gni

@@ -594,6 +594,8 @@ filenames = {
     "shell/common/gin_converters/callback_converter.h",
     "shell/common/gin_converters/content_converter.cc",
     "shell/common/gin_converters/content_converter.h",
+    "shell/common/gin_converters/display_surface_converter.cc",
+    "shell/common/gin_converters/display_surface_converter.h",
     "shell/common/gin_converters/file_dialog_converter.cc",
     "shell/common/gin_converters/file_dialog_converter.h",
     "shell/common/gin_converters/file_path_converter.h",

lib/browser/api/desktop-capturer.ts

@@ -15,7 +15,7 @@ function isValid (options: Electron.SourcesOptions) {
 
 export { isDisplayMediaSystemPickerAvailable };
 
-export async function getSources (args: Electron.SourcesOptions) {
+export async function getSources (args: Electron.SourcesOptions, useSystemPicker: boolean = false) {
   if (!isValid(args)) throw new Error('Invalid options');
 
   const resizableValues = new Map();
@@ -38,7 +38,8 @@ export async function getSources (args: Electron.SourcesOptions) {
     captureWindow,
     captureScreen,
     thumbnailSize,
-    fetchWindowIcons
+    fetchWindowIcons,
+    useSystemPicker,
   };
 
   for (const running of currentlyRunning) {
@@ -80,7 +81,7 @@ export async function getSources (args: Electron.SourcesOptions) {
       resolve(sources);
     };
 
-    capturer.startHandling(captureWindow, captureScreen, thumbnailSize, fetchWindowIcons);
+    capturer.startHandling(captureWindow, captureScreen, thumbnailSize, fetchWindowIcons, useSystemPicker);
   });
 
   currentlyRunning.push({

lib/browser/api/session.ts

@@ -6,7 +6,7 @@ import { desktopCapturer, net } from 'electron/main';
 const { fromPartition, fromPath, Session } = process._linkedBinding('electron_browser_session');
 const { isDisplayMediaSystemPickerAvailable } = process._linkedBinding('electron_browser_desktop_capturer');
 
-async function getNativePickerSource () {
+async function getNativePickerSource (preferredDisplaySurface: string) {
   if (process.platform !== 'darwin') {
     throw new Error('Native system picker option is currently only supported on MacOS');
   }
@@ -16,10 +16,25 @@ async function getNativePickerSource () {
       Note: This is an experimental API; please check the API documentation for updated restrictions`);
   }
 
+  let types: Electron.SourcesOptions["types"];
+  switch (preferredDisplaySurface) {
+    case 'no_preference':
+      types = ['screen', 'window']
+      break;
+    case 'monitor':
+      types = ['screen']
+      break;
+    case 'window':
+      types = ['window']
+      break;
+    default:
+      types = ['screen', 'window']
+  }
+
   // Pass in the needed options for a more native experience
   // screen & windows by default, no thumbnails, since the native picker doesn't return them
   const options: Electron.SourcesOptions = {
-    types: ['screen', 'window'],
+    types,
     thumbnailSize: { width: 0, height: 0 },
     fetchWindowIcons: false
   };
@@ -54,12 +69,12 @@ Session.prototype.fetch = function (input: RequestInfo, init?: RequestInit) {
 Session.prototype.setDisplayMediaRequestHandler = function (handler, opts) {
   if (!handler) return this._setDisplayMediaRequestHandler(handler, opts);
 
-  this._setDisplayMediaRequestHandler(async (req, callback) => {
+  this._setDisplayMediaRequestHandler(async (request, callback) => {
     if (opts && opts.useSystemPicker && isDisplayMediaSystemPickerAvailable()) {
-      return callback({ video: await getNativePickerSource() });
+      return callback({ video: await getNativePickerSource(request.preferredDisplaySurface) });
     }
 
-    return handler(req, callback);
+    return handler(request, callback);
   }, opts);
 };
 

patches/chromium/.patches

@@ -144,3 +144,4 @@ fix_enable_wrap_iter_in_string_view_and_array.patch
 fix_linter_error.patch
 fix_take_snapped_status_into_account_when_showing_a_window.patch
 feat_make_macos_sccontentsharingpicker_work_in_electron.patch
+feat_allow_desktop_capturer_to_return_either_screen_window_or_both.patch

patches/chromium/feat_allow_desktop_capturer_to_return_either_screen_window_or_both.patch

@@ -0,0 +1,105 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Keeley Hammond <khammond@slack-corp.com>
+Date: Tue, 3 Sep 2024 12:38:51 +0100
+Subject: feat: allow desktop capturer to return either screen, window or both
+
+
+diff --git a/chrome/browser/media/webrtc/capture_policy_utils.cc b/chrome/browser/media/webrtc/capture_policy_utils.cc
+index 3e076d38bec2336294d74f0f1a607877d554049a..5e79a54ea9d7581252aeaa0cf961a3c375887e18 100644
+--- a/chrome/browser/media/webrtc/capture_policy_utils.cc
++++ b/chrome/browser/media/webrtc/capture_policy_utils.cc
+@@ -405,6 +405,7 @@ void FilterMediaList(std::vector<DesktopMediaList::Type>& media_types,
+       media_types, [capture_level](const DesktopMediaList::Type& type) {
+         switch (type) {
+           case DesktopMediaList::Type::kNone:
++            // return capture_level < AllowedScreenCaptureLevel::kDesktop;
+             NOTREACHED();
+           // SameOrigin is more restrictive than just Tabs, so as long as
+           // at least SameOrigin is allowed, these entries should stay.
+diff --git a/chrome/browser/media/webrtc/native_desktop_media_list.cc b/chrome/browser/media/webrtc/native_desktop_media_list.cc
+index 68ece50018124992f951557e817a12aa45d65956..7368f26a41907e5004744a07ffe986f036e00b8d 100644
+--- a/chrome/browser/media/webrtc/native_desktop_media_list.cc
++++ b/chrome/browser/media/webrtc/native_desktop_media_list.cc
+@@ -182,14 +182,16 @@ BASE_FEATURE(kWindowCaptureMacV2,
+ 
+ content::DesktopMediaID::Type ConvertToDesktopMediaIDType(
+     DesktopMediaList::Type type) {
++  // LOG(INFO) << "Inside ConvertToDesktopMediaIDType";
+   switch (type) {
+     case DesktopMediaList::Type::kScreen:
+       return content::DesktopMediaID::Type::TYPE_SCREEN;
+     case DesktopMediaList::Type::kWindow:
+       return content::DesktopMediaID::Type::TYPE_WINDOW;
++    case DesktopMediaList::Type::kNone:
++      // return content::DesktopMediaID::Type::TYPE_NONE;
+     case DesktopMediaList::Type::kWebContents:
+     case DesktopMediaList::Type::kCurrentTab:
+-    case DesktopMediaList::Type::kNone:
+       break;
+   }
+   NOTREACHED();
+@@ -533,7 +535,10 @@ NativeDesktopMediaList::Worker::FormatSources(
+   std::vector<SourceDescription> source_descriptions;
+   std::u16string title;
+   for (size_t i = 0; i < sources.size(); ++i) {
++    // LOG(INFO) << "source_type" << source_type;
+     switch (source_type) {
++      // case DesktopMediaID::Type::TYPE_NONE:
++        // continue;
+       case DesktopMediaID::Type::TYPE_SCREEN:
+         // Just in case 'Screen' is inflected depending on the screen number,
+         // use plural formatter.
+diff --git a/content/browser/media/capture/native_screen_capture_picker_mac.mm b/content/browser/media/capture/native_screen_capture_picker_mac.mm
+index 1b8e95f476a032f60ea7e578fac480344924ab50..85693c76c41d9b17bac5f69e99f4f9a4c5003e9c 100644
+--- a/content/browser/media/capture/native_screen_capture_picker_mac.mm
++++ b/content/browser/media/capture/native_screen_capture_picker_mac.mm
+@@ -94,6 +94,7 @@ void Open(DesktopMediaID::Type type,
+             base::OnceCallback<void(Source)> picker_callback,
+             base::OnceCallback<void()> cancel_callback,
+             base::OnceCallback<void()> error_callback) override;
++            // boolean use_system_picker) override;
+   void Close(DesktopMediaID device_id) override;
+   std::unique_ptr<media::VideoCaptureDevice> CreateDevice(
+       const DesktopMediaID& source) override;
+@@ -131,12 +132,13 @@ void Open(DesktopMediaID::Type type,
+     base::OnceCallback<void(Source)> picker_callback,
+     base::OnceCallback<void()> cancel_callback,
+     base::OnceCallback<void()> error_callback) {
++    // boolean use_system_picker) {
+   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+   // Chrome doesn't allow both screens & windows in their picker,
+   // but Electron does - add a check for TYPE_NONE.
+-  CHECK(type == DesktopMediaID::Type::TYPE_SCREEN ||
+-        type == DesktopMediaID::Type::TYPE_WINDOW ||
+-        type == DesktopMediaID::Type::TYPE_NONE);
++  // CHECK(type == DesktopMediaID::Type::TYPE_SCREEN ||
++  //       type == DesktopMediaID::Type::TYPE_WINDOW ||
++  //       type == DesktopMediaID::Type::TYPE_NONE);
+   if (@available(macOS 14.0, *)) {
+     NSNumber* source_id = @(next_id_);
+     auto picker_observer = [[PickerObserver alloc]
+@@ -155,11 +157,24 @@ void Open(DesktopMediaID::Type type,
+     // restart.
+     config.allowsChangingSelectedContent = true;
+     NSNumber* max_stream_count = @(kMaxContentShareCountValue.Get());
++    // LOG(ERROR) << "Type: " << type;
+     // Chrome doesn't allow both screens & windows in their picker,
+     // but Electron does; we patch out the MediaID::Type conditional here
++    // if (type == DesktopMediaID::Type::TYPE_SCREEN) {
++    //   config.allowedPickerModes = SCContentSharingPickerModeSingleDisplay;
++    //   picker.defaultConfiguration = config;
++    //   picker.maximumStreamCount = max_stream_count;
++    //   [picker presentPickerUsingContentStyle:SCShareableContentStyleDisplay];
++    // } else if (type == DesktopMediaID::Type::TYPE_WINDOW) {
++    //   config.allowedPickerModes = SCContentSharingPickerModeSingleWindow;
++    //   picker.defaultConfiguration = config;
++    //   picker.maximumStreamCount = max_stream_count;
++    //   [picker presentPickerUsingContentStyle:SCShareableContentStyleWindow];
++    // } else {
+     picker.defaultConfiguration = config;
+     picker.maximumStreamCount = max_stream_count;
+     [picker present];
++    // }
+   } else {
+     NOTREACHED();
+   }

shell/browser/api/electron_api_desktop_capturer.cc

@@ -9,6 +9,7 @@
 #include <vector>
 
 #include "base/containers/flat_map.h"
+#include "base/logging.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/threading/thread_restrictions.h"
@@ -144,6 +145,21 @@ base::flat_map<int32_t, uint32_t> MonitorAtomIdToDisplayId() {
 }
 #endif
 
+std::unique_ptr<ThumbnailCapturer> MakeScreenAndWindowCapturer() {
+#if BUILDFLAG(IS_MAC)
+  if (ShouldUseThumbnailCapturerMac(DesktopMediaList::Type::kNone)) {
+    LOG(INFO) << "Use the thumbnail capturer";
+    return CreateThumbnailCapturerMac(DesktopMediaList::Type::kNone);
+  }
+#endif  // BUILDFLAG(IS_MAC)
+
+  std::unique_ptr<webrtc::DesktopCapturer> window_capturer =
+      content::desktop_capture::CreateWindowCapturer();
+  return window_capturer ? std::make_unique<DesktopCapturerWrapper>(
+                               std::move(window_capturer))
+                         : nullptr;
+}
+
 std::unique_ptr<ThumbnailCapturer> MakeWindowCapturer() {
 #if BUILDFLAG(IS_MAC)
   if (ShouldUseThumbnailCapturerMac(DesktopMediaList::Type::kWindow)) {
@@ -259,7 +275,8 @@ void DesktopCapturer::DesktopListListener::OnDelegatedSourceListDismissed() {
 void DesktopCapturer::StartHandling(bool capture_window,
                                     bool capture_screen,
                                     const gfx::Size& thumbnail_size,
-                                    bool fetch_window_icons) {
+                                    bool fetch_window_icons,
+                                    bool use_system_picker) {
   fetch_window_icons_ = fetch_window_icons;
 #if BUILDFLAG(IS_WIN)
   if (content::desktop_capture::CreateDesktopCaptureOptions()
@@ -277,31 +294,77 @@ void DesktopCapturer::StartHandling(bool capture_window,
   if (capture_window && capture_screen) {
     // Some capturers like PipeWire support a single capturer for both screens
     // and windows. Use it if possible, treating both as window capture
-    std::unique_ptr<webrtc::DesktopCapturer> desktop_capturer =
-        webrtc::DesktopCapturer::CreateGenericCapturer(
-            content::desktop_capture::CreateDesktopCaptureOptions());
-    auto capturer = desktop_capturer ? std::make_unique<DesktopCapturerWrapper>(
-                                           std::move(desktop_capturer))
-                                     : nullptr;
-    if (capturer && capturer->GetDelegatedSourceListController()) {
+    // std::unique_ptr<webrtc::DesktopCapturer> desktop_capturer =
+    //     webrtc::DesktopCapturer::CreateGenericCapturer(
+    //         content::desktop_capture::CreateDesktopCaptureOptions());
+    LOG(INFO) << "capture_window && capture_screen";
+    // auto capturer = desktop_capturer ? std::make_unique<DesktopCapturerWrapper>(
+    //                                        std::move(desktop_capturer))
+    //                                  : nullptr;
+    // if (capturer && capturer->GetDelegatedSourceListController()) {
+    //   LOG(INFO) << "Inside GetDelegatedSourceListController...";
+    //   capture_screen_ = false;
+    //   capture_window_ = capture_window;
+    //   window_capturer_ = std::make_unique<NativeDesktopMediaList>(
+    //       DesktopMediaList::Type::kWindow, std::move(capturer));
+    //   window_capturer_->SetThumbnailSize(thumbnail_size);
+
+    //   OnceCallback update_callback = base::BindOnce(
+    //       &DesktopCapturer::UpdateSourcesList, weak_ptr_factory_.GetWeakPtr(),
+    //       window_capturer_.get());
+    //   OnceCallback failure_callback = base::BindOnce(
+    //       &DesktopCapturer::HandleFailure, weak_ptr_factory_.GetWeakPtr());
+
+    //   window_listener_ = std::make_unique<DesktopListListener>(
+    //       std::move(update_callback), std::move(failure_callback),
+    //       thumbnail_size.IsEmpty());
+    //   window_capturer_->StartUpdating(window_listener_.get());
+
+    //   return;
+    // }
+
+    // TODO: Add flag for MacOS 15
+    if (IsDisplayMediaSystemPickerAvailable()) {
+      auto capturer = MakeScreenAndWindowCapturer();
+      LOG(INFO) << "Inside the IsDisplayMediaSystemPickerAvailable logic";
       capture_screen_ = false;
       capture_window_ = capture_window;
-      window_capturer_ = std::make_unique<NativeDesktopMediaList>(
-          DesktopMediaList::Type::kWindow, std::move(capturer), true, true);
-      window_capturer_->SetThumbnailSize(thumbnail_size);
+        screen_capturer_ = std::make_unique<NativeDesktopMediaList>(
+            DesktopMediaList::Type::kNone, std::move(capturer), true, true);
+        LOG(INFO) << "Made capturer?";
+        screen_capturer_->SetThumbnailSize(thumbnail_size);
+        LOG(INFO) << "Made thumbnails?";
+        screen_capturer_->ShowDelegatedList();
+        LOG(INFO) << "Showed delegated list?";
+#if BUILDFLAG(IS_MAC)
+        screen_capturer_->skip_next_refresh_ =
+            ShouldUseThumbnailCapturerMac(DesktopMediaList::Type::kNone) ? 2
+                                                                           : 0;
+#endif
 
-      OnceCallback update_callback = base::BindOnce(
-          &DesktopCapturer::UpdateSourcesList, weak_ptr_factory_.GetWeakPtr(),
-          window_capturer_.get());
-      OnceCallback failure_callback = base::BindOnce(
-          &DesktopCapturer::HandleFailure, weak_ptr_factory_.GetWeakPtr());
+        OnceCallback update_callback = base::BindOnce(
+            &DesktopCapturer::UpdateSourcesList, weak_ptr_factory_.GetWeakPtr(),
+            screen_capturer_.get());
+        LOG(INFO) << "Updated source list?";
 
-      window_listener_ = std::make_unique<DesktopListListener>(
-          std::move(update_callback), std::move(failure_callback),
-          thumbnail_size.IsEmpty());
-      window_capturer_->StartUpdating(window_listener_.get());
+        // Needed to force a refresh for the native MacOS Picker
+        OnceCallback wrapped_update_callback = base::BindOnce(
+            &DesktopCapturer::RequestUpdate, weak_ptr_factory_.GetWeakPtr(),
+            screen_capturer_.get(), std::move(update_callback));
 
-      return;
+        if (screen_capturer_->IsSourceListDelegated()) {
+          OnceCallback failure_callback = base::BindOnce(
+              &DesktopCapturer::HandleFailure, weak_ptr_factory_.GetWeakPtr());
+          screen_listener_ = std::make_unique<DesktopListListener>(
+              std::move(wrapped_update_callback), std::move(failure_callback),
+              thumbnail_size.IsEmpty());
+          screen_capturer_->StartUpdating(screen_listener_.get());
+        } else {
+          screen_capturer_->Update(std::move(update_callback),
+                                   /* refresh_thumbnails = */ true);
+        }
+
+        return;
     }
   }
 
@@ -321,6 +384,7 @@ void DesktopCapturer::StartHandling(bool capture_window,
     // Apply the new thumbnail size and restart capture.
     if (capture_window) {
       auto capturer = MakeWindowCapturer();
+      LOG(INFO) << "Hello Capturer";
       if (capturer) {
         window_capturer_ = std::make_unique<NativeDesktopMediaList>(
             DesktopMediaList::Type::kWindow, std::move(capturer), true, true);
@@ -357,6 +421,7 @@ void DesktopCapturer::StartHandling(bool capture_window,
 
     if (capture_screen) {
       auto capturer = MakeScreenCapturer();
+      LOG(INFO) << "Capture Screen...";
       if (capturer) {
         screen_capturer_ = std::make_unique<NativeDesktopMediaList>(
             DesktopMediaList::Type::kScreen, std::move(capturer));

shell/browser/api/electron_api_desktop_capturer.h

@@ -41,7 +41,8 @@ class DesktopCapturer final : public gin::Wrappable<DesktopCapturer>,
   void StartHandling(bool capture_window,
                      bool capture_screen,
                      const gfx::Size& thumbnail_size,
-                     bool fetch_window_icons);
+                     bool fetch_window_icons,
+                     bool use_system_picker);
 
   // gin::Wrappable
   static gin::WrapperInfo kWrapperInfo;

shell/common/gin_converters/display_surface_converter.cc

@@ -0,0 +1,29 @@
+// Copyright (c) 2021 Slack Technologies, LLC.
+// Use of this source code is governed by the MIT license that can be
+// found in the LICENSE file.
+
+#include "shell/common/gin_converters/display_surface_converter.h"
+
+#include "content/public/browser/media_stream_request.h"
+#include "gin/data_object_builder.h"
+#include "shell/common/gin_converters/frame_converter.h"
+#include "shell/common/gin_converters/gurl_converter.h"
+#include "third_party/blink/public/mojom/mediastream/media_stream.mojom.h"
+
+namespace gin {
+
+v8::Local<v8::Value> Converter<blink::mojom::PreferredDisplaySurface>::ToV8(v8::Isolate* isolate,
+                                  blink::mojom::PreferredDisplaySurface type) {
+  switch (type) {
+    case blink::mojom::PreferredDisplaySurface::NO_PREFERENCE:
+      return StringToV8(isolate, "no_preference");
+    case blink::mojom::PreferredDisplaySurface::MONITOR:
+      return StringToV8(isolate, "monitor");
+    case blink::mojom::PreferredDisplaySurface::WINDOW:
+      return StringToV8(isolate, "window");
+    default:
+      return StringToV8(isolate, "unknown");
+  }
+}
+
+}  // namespace gin

shell/common/gin_converters/display_surface_converter.h

@@ -0,0 +1,36 @@
+// Copyright (c) 2024 Slack, Inc.
+// Use of this source code is governed by the MIT license that can be
+// found in the LICENSE file.
+
+#ifndef ELECTRON_SHELL_BROWSER_API_DISPLAY_SOURCE_CONVERTER_H_
+#define ELECTRON_SHELL_BROWSER_API_DISPLAY_SOURCE_CONVERTER_H_
+
+#include <memory>
+#include <string>
+
+#include "gin/converter.h"
+#include "third_party/blink/public/mojom/mediastream/media_stream.mojom.h"
+
+namespace electron {
+
+// cf: 
+// Chrome also allows browser here, but Electron does not.
+enum PreferredDisplaySurface {
+  NO_PREFERENCE,
+  MONITOR,
+  WINDOW,
+};
+
+}  // namespace electron
+
+namespace gin {
+
+template <>
+struct Converter<blink::mojom::PreferredDisplaySurface> {
+  static v8::Local<v8::Value> ToV8(v8::Isolate* isolate,
+                                   blink::mojom::PreferredDisplaySurface val);
+};
+
+}  // namespace gin
+
+#endif  // ELECTRON_SHELL_BROWSER_DISPLAY_SOURCE_CONVERTER_H_

shell/common/gin_converters/media_converter.cc

@@ -4,6 +4,7 @@
 
 #include "shell/common/gin_converters/media_converter.h"
 
+#include "shell/common/gin_converters/display_surface_converter.h"
 #include "content/public/browser/media_stream_request.h"
 #include "content/public/browser/render_frame_host.h"
 #include "gin/data_object_builder.h"
@@ -22,6 +23,7 @@ v8::Local<v8::Value> Converter<content::MediaStreamRequest>::ToV8(
       .Set("frame", rfh)
       .Set("securityOrigin", request.security_origin)
       .Set("userGesture", request.user_gesture)
+      .Set("preferredDisplaySurface", request.preferred_display_surface)
       .Set("videoRequested",
            request.video_type != blink::mojom::MediaStreamType::NO_SERVICE)
       .Set("audioRequested",

typings/internal-electron.d.ts

@@ -266,7 +266,7 @@ declare namespace Electron {
 
 declare namespace ElectronInternal {
   interface DesktopCapturer {
-    startHandling(captureWindow: boolean, captureScreen: boolean, thumbnailSize: Electron.Size, fetchWindowIcons: boolean): void;
+    startHandling(captureWindow: boolean, captureScreen: boolean, thumbnailSize: Electron.Size, fetchWindowIcons: boolean, useSystemPicker: boolean): void;
     _onerror?: (error: string) => void;
     _onfinished?: (sources: Electron.DesktopCapturerSource[], fetchWindowIcons: boolean) => void;
   }