mirror of
https://github.com/electron/electron.git
synced 2026-02-19 03:14:51 -05:00
Compare commits
3 Commits
roller/chr
...
feat/local
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
48950bb970 | ||
|
|
59abf7d2c1 | ||
|
|
9ed9e12418 |
11
BUILD.gn
11
BUILD.gn
@@ -585,6 +585,17 @@ source_set("electron_lib") {
|
||||
]
|
||||
}
|
||||
|
||||
if (enable_prompt_api) {
|
||||
sources += [
|
||||
"shell/browser/ai/proxying_ai_manager.cc",
|
||||
"shell/browser/ai/proxying_ai_manager.h",
|
||||
"shell/utility/ai/utility_ai_language_model.cc",
|
||||
"shell/utility/ai/utility_ai_language_model.h",
|
||||
"shell/utility/ai/utility_ai_manager.cc",
|
||||
"shell/utility/ai/utility_ai_manager.h",
|
||||
]
|
||||
}
|
||||
|
||||
if (is_mac) {
|
||||
deps += [
|
||||
"//components/remote_cocoa/app_shim",
|
||||
|
||||
@@ -12,6 +12,7 @@ buildflag_header("buildflags") {
|
||||
"ENABLE_PDF_VIEWER=$enable_pdf_viewer",
|
||||
"ENABLE_ELECTRON_EXTENSIONS=$enable_electron_extensions",
|
||||
"ENABLE_BUILTIN_SPELLCHECKER=$enable_builtin_spellchecker",
|
||||
"ENABLE_PROMPT_API=$enable_prompt_api",
|
||||
"OVERRIDE_LOCATION_PROVIDER=$enable_fake_location_provider",
|
||||
]
|
||||
|
||||
|
||||
@@ -17,6 +17,9 @@ declare_args() {
|
||||
# Enable Spellchecker support
|
||||
enable_builtin_spellchecker = true
|
||||
|
||||
# Enable Prompt API support.
|
||||
enable_prompt_api = true
|
||||
|
||||
# The version of Electron.
|
||||
# Packagers and vendor builders should set this in gn args to avoid running
|
||||
# the script that reads git tag.
|
||||
|
||||
106
docs/api/language-model.md
Normal file
106
docs/api/language-model.md
Normal file
@@ -0,0 +1,106 @@
|
||||
# LanguageModel
|
||||
|
||||
> Implement local AI language models
|
||||
|
||||
Process: [Utility](../glossary.md#utility-process)
|
||||
|
||||
## Class: LanguageModel
|
||||
|
||||
> Implement local AI language models
|
||||
|
||||
Process: [Utility](../glossary.md#utility-process)
|
||||
|
||||
### `new LanguageModel(initialState)`
|
||||
|
||||
* `initialState` Object
|
||||
* `inputUsage` number
|
||||
* `inputQuota` number
|
||||
* `temperature` number
|
||||
* `topK` number
|
||||
|
||||
> [!NOTE]
|
||||
> Do not use this constructor directly outside of the class itself, as it will not be properly connected to the `localAIHandler`
|
||||
|
||||
### Static Methods
|
||||
|
||||
The `LanguageModel` class has the following static methods:
|
||||
|
||||
#### `LanguageModel.create([options])` _Experimental_
|
||||
|
||||
* `options` [LanguageModelCreateOptions](structures/language-model-create-options.md) (optional)
|
||||
|
||||
Returns `Promise<LanguageModel>`
|
||||
|
||||
#### `LanguageModel.availability([options])` _Experimental_
|
||||
|
||||
* `options` [LanguageModelCreateCoreOptions](structures/language-model-create-core-options.md) (optional)
|
||||
|
||||
Returns `Promise<string>`
|
||||
|
||||
Determines the availability of the language model and returns one of the following strings:
|
||||
|
||||
* `available`
|
||||
* `downloadable`
|
||||
* `downloading`
|
||||
* `unavailable`
|
||||
|
||||
#### `LanguageModel.params()` _Experimental_
|
||||
|
||||
Returns `Promise<LanguageModelParams | null>`
|
||||
|
||||
### Instance Properties
|
||||
|
||||
The following properties are available on instances of `LanguageModel`:
|
||||
|
||||
#### `languageModel.inputUsage` _Readonly_ _Experimental_
|
||||
|
||||
A `number` representing TODO.
|
||||
|
||||
#### `languageModel.inputQuota` _Readonly_ _Experimental_
|
||||
|
||||
A `number` representing TODO.
|
||||
|
||||
#### `languageModel.topK` _Readonly_ _Experimental_
|
||||
|
||||
A `number` representing TODO.
|
||||
|
||||
#### `languageModel.temperature` _Readonly_ _Experimental_
|
||||
|
||||
A `number` representing TODO.
|
||||
|
||||
### Instance Methods
|
||||
|
||||
The following methods are available on instances of `LanguageModel`:
|
||||
|
||||
#### `languageModel.prompt(input, [options])` _Experimental_
|
||||
|
||||
* `input` [LanguageModelMessage[]](structures/language-model-message.md) | string
|
||||
* `options` [LanguageModelPromptOptions](structures/language-model-prompt-options.md) (optional)
|
||||
|
||||
<!-- TODO: This types as the wrong ReadableStream, we want the web ReadableStream -->
|
||||
|
||||
Returns `Promise<string> | ReadableStream<string>`
|
||||
|
||||
#### `languageModel.append(input, [options])` _Experimental_
|
||||
|
||||
* `input` [LanguageModelMessage[]](structures/language-model-message.md) | string
|
||||
* `options` [LanguageModelAppendOptions](structures/language-model-append-options.md) (optional)
|
||||
|
||||
Returns `Promise<undefined>`
|
||||
|
||||
#### `languageModel.measureInputUsage(input, [options])` _Experimental_
|
||||
|
||||
* `input` [LanguageModelMessage[]](structures/language-model-message.md) | string
|
||||
* `options` [LanguageModelPromptOptions](structures/language-model-prompt-options.md) (optional)
|
||||
|
||||
Returns `Promise<number>`
|
||||
|
||||
#### `languageModel.clone([options])` _Experimental_
|
||||
|
||||
* `options` [LanguageModelCloneOptions](structures/language-model-clone-options.md) (optional)
|
||||
|
||||
Returns `Promise<LanguageModel>`
|
||||
|
||||
#### `languageModel.destroy()` _Experimental_
|
||||
|
||||
Destroys the model
|
||||
21
docs/api/local-ai-handler.md
Normal file
21
docs/api/local-ai-handler.md
Normal file
@@ -0,0 +1,21 @@
|
||||
# localAIHandler
|
||||
|
||||
> Proxy Built-in AI APIs to a local LLM implementation
|
||||
|
||||
Process: [Utility](../glossary.md#utility-process)
|
||||
|
||||
This module is intended to be used by a script registered to a session via
|
||||
[`ses.registerLocalAIHandlerScript(options)`](./session.md#sesregisterlocalaihandlerhandler-experimental)
|
||||
|
||||
## Methods
|
||||
|
||||
The `localAIHandler` module has the following methods:
|
||||
|
||||
#### `localAIHandler.setPromptAPIHandler(handler)` _Experimental_
|
||||
|
||||
* `handler` Function\<Promise\<typeof [LanguageModel](language-model.md)\>\> | null
|
||||
* `details` Object
|
||||
* `webContentsId` (Integer | null) - The [unique id](web-contents.md#contentsid-readonly) of
|
||||
the [WebContents](web-contents.md) calling the Prompt API. The WebContents id may be null
|
||||
if the Prompt API is called from a service worker or shared worker.
|
||||
* `securityOrigin` string - Origin of the page calling the Prompt API.
|
||||
@@ -1627,6 +1627,12 @@ This method clears more types of data and is more thorough than the
|
||||
|
||||
For more information, refer to Chromium's [`BrowsingDataRemover` interface][browsing-data-remover].
|
||||
|
||||
#### `ses.registerLocalAIHandler(handler)` _Experimental_
|
||||
|
||||
* `handler` [UtilityProcess](utility-process.md#class-utilityprocess) | null
|
||||
|
||||
Registers a local AI handler `UtilityProcess`. To clear the handler, call `registerLocalAIHandler(null)`.
|
||||
|
||||
### Instance Properties
|
||||
|
||||
The following properties are available on instances of `Session`:
|
||||
|
||||
3
docs/api/structures/language-model-append-options.md
Normal file
3
docs/api/structures/language-model-append-options.md
Normal file
@@ -0,0 +1,3 @@
|
||||
# LanguageModelAppendOptions Object
|
||||
|
||||
* `signal` [AbortSignal](https://nodejs.org/api/globals.html#globals_class_abortsignal) (optional)
|
||||
3
docs/api/structures/language-model-clone-options.md
Normal file
3
docs/api/structures/language-model-clone-options.md
Normal file
@@ -0,0 +1,3 @@
|
||||
# LanguageModelCloneOptions Object
|
||||
|
||||
* `signal` [AbortSignal](https://nodejs.org/api/globals.html#globals_class_abortsignal) (optional)
|
||||
@@ -0,0 +1,6 @@
|
||||
# LanguageModelCreateCoreOptions Object
|
||||
|
||||
* `topK` number (optional)
|
||||
* `temperature` number (optional)
|
||||
* `expectedInputs` [LanguageModelExpected[]](language-model-expected.md) (optional)
|
||||
* `expectedOutputs` [LanguageModelExpected[]](language-model-expected.md) (optional)
|
||||
4
docs/api/structures/language-model-create-options.md
Normal file
4
docs/api/structures/language-model-create-options.md
Normal file
@@ -0,0 +1,4 @@
|
||||
# LanguageModelCreateOptions Object extends `LanguageModelCreateCoreOptions`
|
||||
|
||||
* `signal` [AbortSignal](https://nodejs.org/api/globals.html#globals_class_abortsignal) (optional)
|
||||
* `initialPrompts` [LanguageModelMessage[]](language-model-message.md) (optional)
|
||||
7
docs/api/structures/language-model-expected.md
Normal file
7
docs/api/structures/language-model-expected.md
Normal file
@@ -0,0 +1,7 @@
|
||||
# LanguageModelExpected Object
|
||||
|
||||
* `type` string - Can be one of the following values:
|
||||
* `text`
|
||||
* `image`
|
||||
* `audio`
|
||||
* `languages` string[] (optional)
|
||||
7
docs/api/structures/language-model-message-content.md
Normal file
7
docs/api/structures/language-model-message-content.md
Normal file
@@ -0,0 +1,7 @@
|
||||
# LanguageModelMessageContent Object
|
||||
|
||||
* `type` string - Can be one of the following values:
|
||||
* `text`
|
||||
* `image`
|
||||
* `audio`
|
||||
* `value` ArrayBuffer | string
|
||||
8
docs/api/structures/language-model-message.md
Normal file
8
docs/api/structures/language-model-message.md
Normal file
@@ -0,0 +1,8 @@
|
||||
# LanguageModelMessage Object
|
||||
|
||||
* `role` string - Can be one of the following values:
|
||||
* `system`
|
||||
* `user`
|
||||
* `assistant`
|
||||
* `content` [LanguageModelMessageContent[]](language-model-message-content.md) | string
|
||||
* `prefix` boolean (optional)
|
||||
6
docs/api/structures/language-model-params.md
Normal file
6
docs/api/structures/language-model-params.md
Normal file
@@ -0,0 +1,6 @@
|
||||
# LanguageModelParams Object
|
||||
|
||||
* `defaultTopK` number _Readonly_
|
||||
* `maxTopK` number _Readonly_
|
||||
* `defaultTemperature` number _Readonly_
|
||||
* `maxTemperature` number _Readonly_
|
||||
4
docs/api/structures/language-model-prompt-options.md
Normal file
4
docs/api/structures/language-model-prompt-options.md
Normal file
@@ -0,0 +1,4 @@
|
||||
# LanguageModelPromptOptions Object
|
||||
|
||||
* `responseConstraint` Object (optional)
|
||||
* `signal` [AbortSignal](https://nodejs.org/api/globals.html#globals_class_abortsignal) (optional)
|
||||
@@ -30,6 +30,8 @@ auto_filenames = {
|
||||
"docs/api/ipc-main-service-worker.md",
|
||||
"docs/api/ipc-main.md",
|
||||
"docs/api/ipc-renderer.md",
|
||||
"docs/api/language-model.md",
|
||||
"docs/api/local-ai-handler.md",
|
||||
"docs/api/menu-item.md",
|
||||
"docs/api/menu.md",
|
||||
"docs/api/message-channel-main.md",
|
||||
@@ -106,6 +108,15 @@ auto_filenames = {
|
||||
"docs/api/structures/jump-list-item.md",
|
||||
"docs/api/structures/keyboard-event.md",
|
||||
"docs/api/structures/keyboard-input-event.md",
|
||||
"docs/api/structures/language-model-append-options.md",
|
||||
"docs/api/structures/language-model-clone-options.md",
|
||||
"docs/api/structures/language-model-create-core-options.md",
|
||||
"docs/api/structures/language-model-create-options.md",
|
||||
"docs/api/structures/language-model-expected.md",
|
||||
"docs/api/structures/language-model-message-content.md",
|
||||
"docs/api/structures/language-model-message.md",
|
||||
"docs/api/structures/language-model-params.md",
|
||||
"docs/api/structures/language-model-prompt-options.md",
|
||||
"docs/api/structures/media-access-permission-request.md",
|
||||
"docs/api/structures/memory-info.md",
|
||||
"docs/api/structures/memory-usage-details.md",
|
||||
@@ -385,6 +396,8 @@ auto_filenames = {
|
||||
"lib/common/init.ts",
|
||||
"lib/common/webpack-globals-provider.ts",
|
||||
"lib/utility/api/exports/electron.ts",
|
||||
"lib/utility/api/language-model.ts",
|
||||
"lib/utility/api/local-ai-handler.ts",
|
||||
"lib/utility/api/module-list.ts",
|
||||
"lib/utility/api/net.ts",
|
||||
"lib/utility/init.ts",
|
||||
|
||||
@@ -737,6 +737,8 @@ filenames = {
|
||||
"shell/services/node/node_service.h",
|
||||
"shell/services/node/parent_port.cc",
|
||||
"shell/services/node/parent_port.h",
|
||||
"shell/utility/api/electron_api_local_ai_handler.cc",
|
||||
"shell/utility/api/electron_api_local_ai_handler.h",
|
||||
"shell/utility/electron_content_utility_client.cc",
|
||||
"shell/utility/electron_content_utility_client.h",
|
||||
]
|
||||
|
||||
@@ -2,7 +2,7 @@ import { fetchWithSession } from '@electron/internal/browser/api/net-fetch';
|
||||
import { addIpcDispatchListeners } from '@electron/internal/browser/ipc-dispatch';
|
||||
import * as deprecate from '@electron/internal/common/deprecate';
|
||||
|
||||
import { net } from 'electron/main';
|
||||
import { net, type UtilityProcess } from 'electron/main';
|
||||
|
||||
const { fromPartition, fromPath, Session } = process._linkedBinding('electron_browser_session');
|
||||
const { isDisplayMediaSystemPickerAvailable } = process._linkedBinding('electron_browser_desktop_capturer');
|
||||
@@ -111,6 +111,10 @@ Session.prototype.removeExtension = deprecate.moveAPI(
|
||||
'session.extensions.removeExtension'
|
||||
);
|
||||
|
||||
Session.prototype.registerLocalAIHandler = function (handler: UtilityProcess | null) {
|
||||
return this._registerLocalAIHandler(handler !== null ? (handler as any)._unwrapHandle() : null);
|
||||
};
|
||||
|
||||
export default {
|
||||
fromPartition,
|
||||
fromPath,
|
||||
|
||||
@@ -131,6 +131,10 @@ class ForkUtilityProcess extends EventEmitter implements Electron.UtilityProcess
|
||||
return this.#stderr;
|
||||
}
|
||||
|
||||
_unwrapHandle () {
|
||||
return this.#handle;
|
||||
}
|
||||
|
||||
postMessage (message: any, transfer?: MessagePortMain[]) {
|
||||
if (Array.isArray(transfer)) {
|
||||
transfer = transfer.map((o: any) => o instanceof MessagePortMain ? o._internalPort : o);
|
||||
|
||||
58
lib/utility/api/language-model.ts
Normal file
58
lib/utility/api/language-model.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
interface LanguageModelConstructorValues {
|
||||
inputUsage: number;
|
||||
inputQuota: number;
|
||||
topK: number;
|
||||
temperature: number;
|
||||
}
|
||||
|
||||
export default class LanguageModel implements Electron.LanguageModel {
|
||||
readonly inputUsage: number;
|
||||
readonly inputQuota: number;
|
||||
readonly topK: number;
|
||||
readonly temperature: number;
|
||||
|
||||
constructor (values: LanguageModelConstructorValues) {
|
||||
this.inputUsage = values.inputUsage;
|
||||
this.inputQuota = values.inputQuota;
|
||||
this.topK = values.topK;
|
||||
this.temperature = values.temperature;
|
||||
}
|
||||
|
||||
static async create (): Promise<LanguageModel> {
|
||||
return new LanguageModel({
|
||||
inputUsage: 0,
|
||||
inputQuota: 0,
|
||||
topK: 0,
|
||||
temperature: 0
|
||||
});
|
||||
}
|
||||
|
||||
static async availability () {
|
||||
return 'available';
|
||||
}
|
||||
|
||||
static async params () {
|
||||
return null;
|
||||
}
|
||||
|
||||
async prompt () {
|
||||
return '';
|
||||
}
|
||||
|
||||
async append (): Promise<undefined> {}
|
||||
|
||||
async measureInputUsage () {
|
||||
return 0;
|
||||
}
|
||||
|
||||
async clone () {
|
||||
return new LanguageModel({
|
||||
inputUsage: this.inputUsage,
|
||||
inputQuota: this.inputQuota,
|
||||
topK: this.topK,
|
||||
temperature: this.temperature
|
||||
});
|
||||
}
|
||||
|
||||
destroy () {}
|
||||
}
|
||||
3
lib/utility/api/local-ai-handler.ts
Normal file
3
lib/utility/api/local-ai-handler.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
const binding = process._linkedBinding('electron_utility_local_ai_handler');
|
||||
|
||||
export const setPromptAPIHandler = binding.setPromptAPIHandler;
|
||||
@@ -1,5 +1,7 @@
|
||||
// Utility side modules, please sort alphabetically.
|
||||
export const utilityNodeModuleList: ElectronInternal.ModuleEntry[] = [
|
||||
{ name: 'localAIHandler', loader: () => require('./local-ai-handler') },
|
||||
{ name: 'LanguageModel', loader: () => require('./language-model') },
|
||||
{ name: 'net', loader: () => require('./net') },
|
||||
{ name: 'systemPreferences', loader: () => require('@electron/internal/browser/api/system-preferences') }
|
||||
];
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import LanguageModel from '@electron/internal/utility/api/language-model';
|
||||
import { ParentPort } from '@electron/internal/utility/parent-port';
|
||||
|
||||
import { EventEmitter } from 'events';
|
||||
import { ReadableStream } from 'stream/web';
|
||||
import { pathToFileURL } from 'url';
|
||||
|
||||
const v8Util = process._linkedBinding('electron_common_v8_util');
|
||||
@@ -10,6 +12,11 @@ const entryScript: string = v8Util.getHiddenValue(process, '_serviceStartupScrip
|
||||
// we need to restore it here.
|
||||
process.argv.splice(1, 1, entryScript);
|
||||
|
||||
// These are used by C++ to more easily identify these objects.
|
||||
v8Util.setHiddenValue(global, 'isReadableStream', (val: unknown) => val instanceof ReadableStream);
|
||||
v8Util.setHiddenValue(global, 'isLanguageModel', (val: unknown) => val instanceof LanguageModel);
|
||||
v8Util.setHiddenValue(global, 'isLanguageModelClass', (val: any) => Object.is(val, LanguageModel) || val.prototype instanceof LanguageModel);
|
||||
|
||||
// Import common settings.
|
||||
require('@electron/internal/common/init');
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"@electron/fiddle-core": "^1.3.4",
|
||||
"@electron/github-app-auth": "^2.2.1",
|
||||
"@electron/lint-roller": "^3.1.2",
|
||||
"@electron/typescript-definitions": "^9.1.2",
|
||||
"@electron/typescript-definitions": "^9.1.5",
|
||||
"@octokit/rest": "^20.1.2",
|
||||
"@primer/octicons": "^10.0.0",
|
||||
"@types/minimist": "^1.2.5",
|
||||
|
||||
196
shell/browser/ai/proxying_ai_manager.cc
Normal file
196
shell/browser/ai/proxying_ai_manager.cc
Normal file
@@ -0,0 +1,196 @@
|
||||
// Copyright (c) 2025 Microsoft, Inc.
|
||||
// Use of this source code is governed by the MIT license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#include "shell/browser/ai/proxying_ai_manager.h"
|
||||
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
|
||||
#include "base/notimplemented.h"
|
||||
#include "content/public/browser/browser_context.h"
|
||||
#include "content/public/browser/render_frame_host.h"
|
||||
#include "content/public/browser/weak_document_ptr.h"
|
||||
#include "mojo/public/cpp/bindings/callback_helpers.h"
|
||||
#include "shell/browser/api/electron_api_session.h"
|
||||
#include "shell/browser/api/electron_api_web_contents.h"
|
||||
#include "shell/browser/session_preferences.h"
|
||||
#include "third_party/blink/public/mojom/ai/ai_common.mojom.h"
|
||||
#include "third_party/blink/public/mojom/ai/ai_language_model.mojom.h"
|
||||
|
||||
namespace electron {
|
||||
|
||||
ProxyingAIManager::ProxyingAIManager(content::BrowserContext* browser_context,
|
||||
content::RenderFrameHost* rfh)
|
||||
: browser_context_(browser_context),
|
||||
rfh_(rfh ? rfh->GetWeakDocumentPtr() : content::WeakDocumentPtr()) {}
|
||||
|
||||
ProxyingAIManager::~ProxyingAIManager() = default;
|
||||
|
||||
void ProxyingAIManager::AddReceiver(
|
||||
mojo::PendingReceiver<blink::mojom::AIManager> receiver) {
|
||||
receivers_.Add(this, std::move(receiver));
|
||||
}
|
||||
|
||||
const mojo::Remote<blink::mojom::AIManager>&
|
||||
ProxyingAIManager::GetAIManagerRemote(const SessionPreferences& session_prefs) {
|
||||
if (!ai_manager_remote_.is_bound()) {
|
||||
auto* local_ai_handler = session_prefs.GetLocalAIHandler().get();
|
||||
|
||||
if (local_ai_handler) {
|
||||
auto* rfh = rfh_.AsRenderFrameHostIfValid();
|
||||
DCHECK(rfh);
|
||||
|
||||
auto* web_contents = electron::api::WebContents::From(
|
||||
content::WebContents::FromRenderFrameHost(rfh));
|
||||
std::optional<int32_t> web_contents_id;
|
||||
|
||||
if (web_contents) {
|
||||
web_contents_id = web_contents->ID();
|
||||
}
|
||||
|
||||
local_ai_handler->BindAIManager(
|
||||
web_contents_id, rfh->GetLastCommittedOrigin(),
|
||||
ai_manager_remote_.BindNewPipeAndPassReceiver());
|
||||
}
|
||||
} else {
|
||||
// Developer may have unregistered the local AI handler
|
||||
// TODO - This should really happen the moment they unregister
|
||||
// it so that in-progress calls to it get killed off
|
||||
auto* local_ai_handler = session_prefs.GetLocalAIHandler().get();
|
||||
|
||||
if (!local_ai_handler) {
|
||||
ai_manager_remote_.reset();
|
||||
}
|
||||
}
|
||||
|
||||
return ai_manager_remote_;
|
||||
}
|
||||
|
||||
void ProxyingAIManager::CanCreateLanguageModel(
|
||||
blink::mojom::AILanguageModelCreateOptionsPtr options,
|
||||
CanCreateLanguageModelCallback callback) {
|
||||
auto* session_prefs =
|
||||
SessionPreferences::FromBrowserContext(browser_context_);
|
||||
DCHECK(session_prefs);
|
||||
|
||||
// Default to unavailable. This ensures the callback is always invoked
|
||||
// even if there is no registered utility process handler, or the
|
||||
// process crashes.
|
||||
auto cb = mojo::WrapCallbackWithDefaultInvokeIfNotRun(
|
||||
std::move(callback),
|
||||
blink::mojom::ModelAvailabilityCheckResult::kUnavailableUnknown);
|
||||
|
||||
// Proxy the call through to the utility process
|
||||
auto& ai_manager = GetAIManagerRemote(*session_prefs);
|
||||
|
||||
if (ai_manager.is_bound()) {
|
||||
ai_manager->CanCreateLanguageModel(std::move(options), std::move(cb));
|
||||
}
|
||||
}
|
||||
|
||||
void ProxyingAIManager::CreateLanguageModel(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateLanguageModelClient>
|
||||
client,
|
||||
blink::mojom::AILanguageModelCreateOptionsPtr options) {
|
||||
auto* session_prefs =
|
||||
SessionPreferences::FromBrowserContext(browser_context_);
|
||||
DCHECK(session_prefs);
|
||||
|
||||
// Proxy the call through to the utility process
|
||||
auto& ai_manager = GetAIManagerRemote(*session_prefs);
|
||||
|
||||
if (!ai_manager.is_bound()) {
|
||||
mojo::Remote<blink::mojom::AIManagerCreateLanguageModelClient>
|
||||
client_remote(std::move(client));
|
||||
client_remote->OnError(
|
||||
blink::mojom::AIManagerCreateClientError::kUnableToCreateSession,
|
||||
/*quota_error_info=*/nullptr);
|
||||
return;
|
||||
}
|
||||
|
||||
ai_manager->CreateLanguageModel(std::move(client), std::move(options));
|
||||
|
||||
// TODO - Implement language model creation logic
|
||||
// TODO - Does there need to be a proxying AILanguageModel impl?
|
||||
}
|
||||
|
||||
void ProxyingAIManager::CanCreateSummarizer(
|
||||
blink::mojom::AISummarizerCreateOptionsPtr options,
|
||||
CanCreateSummarizerCallback callback) {
|
||||
std::move(callback).Run(
|
||||
blink::mojom::ModelAvailabilityCheckResult::kUnavailableUnknown);
|
||||
}
|
||||
|
||||
void ProxyingAIManager::CreateSummarizer(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateSummarizerClient> client,
|
||||
blink::mojom::AISummarizerCreateOptionsPtr options) {
|
||||
NOTIMPLEMENTED();
|
||||
}
|
||||
|
||||
void ProxyingAIManager::GetLanguageModelParams(
|
||||
GetLanguageModelParamsCallback callback) {
|
||||
auto* session_prefs =
|
||||
SessionPreferences::FromBrowserContext(browser_context_);
|
||||
DCHECK(session_prefs);
|
||||
|
||||
// Default to null. This ensures the callback is always invoked
|
||||
// even if there is no registered utility process handler, or the
|
||||
// process crashes.
|
||||
auto cb =
|
||||
mojo::WrapCallbackWithDefaultInvokeIfNotRun(std::move(callback), nullptr);
|
||||
|
||||
// Proxy the call through to the utility process
|
||||
auto& ai_manager = GetAIManagerRemote(*session_prefs);
|
||||
|
||||
if (ai_manager.is_bound()) {
|
||||
ai_manager->GetLanguageModelParams(std::move(cb));
|
||||
}
|
||||
}
|
||||
|
||||
void ProxyingAIManager::CanCreateWriter(
|
||||
blink::mojom::AIWriterCreateOptionsPtr options,
|
||||
CanCreateWriterCallback callback) {
|
||||
std::move(callback).Run(
|
||||
blink::mojom::ModelAvailabilityCheckResult::kUnavailableUnknown);
|
||||
}
|
||||
|
||||
void ProxyingAIManager::CreateWriter(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateWriterClient> client,
|
||||
blink::mojom::AIWriterCreateOptionsPtr options) {
|
||||
NOTIMPLEMENTED();
|
||||
}
|
||||
|
||||
void ProxyingAIManager::CanCreateRewriter(
|
||||
blink::mojom::AIRewriterCreateOptionsPtr options,
|
||||
CanCreateRewriterCallback callback) {
|
||||
std::move(callback).Run(
|
||||
blink::mojom::ModelAvailabilityCheckResult::kUnavailableUnknown);
|
||||
}
|
||||
|
||||
void ProxyingAIManager::CreateRewriter(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateRewriterClient> client,
|
||||
blink::mojom::AIRewriterCreateOptionsPtr options) {
|
||||
NOTIMPLEMENTED();
|
||||
}
|
||||
|
||||
void ProxyingAIManager::CanCreateProofreader(
|
||||
blink::mojom::AIProofreaderCreateOptionsPtr options,
|
||||
CanCreateProofreaderCallback callback) {
|
||||
std::move(callback).Run(
|
||||
blink::mojom::ModelAvailabilityCheckResult::kUnavailableUnknown);
|
||||
}
|
||||
|
||||
void ProxyingAIManager::CreateProofreader(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateProofreaderClient> client,
|
||||
blink::mojom::AIProofreaderCreateOptionsPtr options) {
|
||||
NOTIMPLEMENTED();
|
||||
}
|
||||
|
||||
void ProxyingAIManager::AddModelDownloadProgressObserver(
|
||||
mojo::PendingRemote<blink::mojom::ModelDownloadProgressObserver>
|
||||
observer_remote) {
|
||||
NOTIMPLEMENTED();
|
||||
}
|
||||
|
||||
} // namespace electron
|
||||
97
shell/browser/ai/proxying_ai_manager.h
Normal file
97
shell/browser/ai/proxying_ai_manager.h
Normal file
@@ -0,0 +1,97 @@
|
||||
// Copyright (c) 2025 Microsoft, Inc.
|
||||
// Use of this source code is governed by the MIT license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef ELECTRON_SHELL_BROWSER_AI_PROXYING_AI_MANAGER_H_
|
||||
#define ELECTRON_SHELL_BROWSER_AI_PROXYING_AI_MANAGER_H_
|
||||
|
||||
#include "base/memory/weak_ptr.h"
|
||||
#include "base/supports_user_data.h"
|
||||
#include "content/public/browser/weak_document_ptr.h"
|
||||
#include "mojo/public/cpp/bindings/pending_receiver.h"
|
||||
#include "mojo/public/cpp/bindings/pending_remote.h"
|
||||
#include "mojo/public/cpp/bindings/receiver_set.h"
|
||||
#include "shell/browser/session_preferences.h"
|
||||
#include "third_party/blink/public/mojom/ai/ai_language_model.mojom.h"
|
||||
#include "third_party/blink/public/mojom/ai/ai_manager.mojom.h"
|
||||
|
||||
namespace content {
|
||||
class BrowserContext;
|
||||
class RenderFrameHost;
|
||||
} // namespace content
|
||||
|
||||
namespace electron {
|
||||
|
||||
// Owned by the host of the document / service worker via `SupportUserData`.
|
||||
// The browser-side implementation of `blink::mojom::AIManager`, which
|
||||
// proxies requests to a utility process if the session has a registered
|
||||
// handler.
|
||||
class ProxyingAIManager : public base::SupportsUserData::Data,
|
||||
public blink::mojom::AIManager {
|
||||
public:
|
||||
ProxyingAIManager(content::BrowserContext* browser_context,
|
||||
content::RenderFrameHost* rfh);
|
||||
ProxyingAIManager(const ProxyingAIManager&) = delete;
|
||||
ProxyingAIManager& operator=(const ProxyingAIManager&) = delete;
|
||||
|
||||
~ProxyingAIManager() override;
|
||||
|
||||
void AddReceiver(mojo::PendingReceiver<blink::mojom::AIManager> receiver);
|
||||
|
||||
private:
|
||||
// Lazily bind the AIManager remote so that the developer can
|
||||
// set the local AI handler after this class is already created
|
||||
[[nodiscard]] const mojo::Remote<blink::mojom::AIManager>& GetAIManagerRemote(
|
||||
const SessionPreferences& session_prefs);
|
||||
|
||||
// `blink::mojom::AIManager` implementation.
|
||||
void CanCreateLanguageModel(
|
||||
blink::mojom::AILanguageModelCreateOptionsPtr options,
|
||||
CanCreateLanguageModelCallback callback) override;
|
||||
void CreateLanguageModel(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateLanguageModelClient>
|
||||
client,
|
||||
blink::mojom::AILanguageModelCreateOptionsPtr options) override;
|
||||
void CanCreateSummarizer(blink::mojom::AISummarizerCreateOptionsPtr options,
|
||||
CanCreateSummarizerCallback callback) override;
|
||||
void CreateSummarizer(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateSummarizerClient> client,
|
||||
blink::mojom::AISummarizerCreateOptionsPtr options) override;
|
||||
void GetLanguageModelParams(GetLanguageModelParamsCallback callback) override;
|
||||
void CanCreateWriter(blink::mojom::AIWriterCreateOptionsPtr options,
|
||||
CanCreateWriterCallback callback) override;
|
||||
void CreateWriter(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateWriterClient> client,
|
||||
blink::mojom::AIWriterCreateOptionsPtr options) override;
|
||||
void CanCreateRewriter(blink::mojom::AIRewriterCreateOptionsPtr options,
|
||||
CanCreateRewriterCallback callback) override;
|
||||
void CreateRewriter(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateRewriterClient> client,
|
||||
blink::mojom::AIRewriterCreateOptionsPtr options) override;
|
||||
void CanCreateProofreader(blink::mojom::AIProofreaderCreateOptionsPtr options,
|
||||
CanCreateProofreaderCallback callback) override;
|
||||
void CreateProofreader(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateProofreaderClient>
|
||||
client,
|
||||
blink::mojom::AIProofreaderCreateOptionsPtr options) override;
|
||||
void AddModelDownloadProgressObserver(
|
||||
mojo::PendingRemote<blink::mojom::ModelDownloadProgressObserver>
|
||||
observer_remote) override;
|
||||
|
||||
mojo::ReceiverSet<blink::mojom::AIManager> receivers_;
|
||||
|
||||
raw_ptr<content::BrowserContext> browser_context_;
|
||||
|
||||
content::WeakDocumentPtr rfh_;
|
||||
|
||||
// TODO - How to handle the case where the developer has switched to a
|
||||
// different handler?
|
||||
mojo::Remote<blink::mojom::AIManager> ai_manager_remote_;
|
||||
|
||||
// TODO(dsanders11) - This is unused at the moment
|
||||
base::WeakPtrFactory<ProxyingAIManager> weak_ptr_factory_{this};
|
||||
};
|
||||
|
||||
} // namespace electron
|
||||
|
||||
#endif // ELECTRON_SHELL_BROWSER_AI_PROXYING_AI_MANAGER_H_
|
||||
@@ -1548,6 +1548,26 @@ v8::Local<v8::Value> Session::ClearData(gin_helper::ErrorThrower thrower,
|
||||
return promise_handle;
|
||||
}
|
||||
|
||||
void Session::RegisterLocalAIHandler(gin_helper::ErrorThrower thrower,
|
||||
v8::Local<v8::Value> val) {
|
||||
auto* isolate = JavascriptEnvironment::GetIsolate();
|
||||
gin_helper::Handle<UtilityProcessWrapper> handler;
|
||||
|
||||
if (!(val->IsNull() || gin::ConvertFromV8(isolate, val, &handler))) {
|
||||
thrower.ThrowTypeError("Must pass null or UtilityProcess");
|
||||
return;
|
||||
}
|
||||
|
||||
auto* prefs = SessionPreferences::FromBrowserContext(browser_context());
|
||||
DCHECK(prefs);
|
||||
|
||||
if (!handler.IsEmpty()) {
|
||||
prefs->SetLocalAIHandler(handler->GetWeakPtr());
|
||||
} else {
|
||||
prefs->SetLocalAIHandler(nullptr);
|
||||
}
|
||||
}
|
||||
|
||||
#if BUILDFLAG(ENABLE_BUILTIN_SPELLCHECKER)
|
||||
base::Value Session::GetSpellCheckerLanguages() {
|
||||
return browser_context_->prefs()
|
||||
@@ -1828,6 +1848,7 @@ void Session::FillObjectTemplate(v8::Isolate* isolate,
|
||||
.SetMethod("setCodeCachePath", &Session::SetCodeCachePath)
|
||||
.SetMethod("clearCodeCaches", &Session::ClearCodeCaches)
|
||||
.SetMethod("clearData", &Session::ClearData)
|
||||
.SetMethod("_registerLocalAIHandler", &Session::RegisterLocalAIHandler)
|
||||
.SetProperty("cookies", &Session::Cookies)
|
||||
.SetProperty("extensions", &Session::Extensions)
|
||||
.SetProperty("netLog", &Session::NetLog)
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
#include "gin/wrappable.h"
|
||||
#include "services/network/public/mojom/host_resolver.mojom-forward.h"
|
||||
#include "services/network/public/mojom/ssl_config.mojom-forward.h"
|
||||
#include "shell/browser/api/electron_api_utility_process.h"
|
||||
#include "shell/browser/api/ipc_dispatcher.h"
|
||||
#include "shell/browser/event_emitter_mixin.h"
|
||||
#include "shell/browser/net/resolve_proxy_helper.h"
|
||||
@@ -170,6 +171,8 @@ class Session final : public gin::Wrappable<Session>,
|
||||
v8::Local<v8::Promise> ClearCodeCaches(const gin_helper::Dictionary& options);
|
||||
v8::Local<v8::Value> ClearData(gin_helper::ErrorThrower thrower,
|
||||
gin::Arguments* args);
|
||||
void RegisterLocalAIHandler(gin_helper::ErrorThrower thrower,
|
||||
v8::Local<v8::Value> val);
|
||||
#if BUILDFLAG(ENABLE_BUILTIN_SPELLCHECKER)
|
||||
base::Value GetSpellCheckerLanguages();
|
||||
void SetSpellCheckerLanguages(gin_helper::ErrorThrower thrower,
|
||||
|
||||
@@ -45,6 +45,10 @@
|
||||
#include "base/win/windows_types.h"
|
||||
#endif
|
||||
|
||||
#if BUILDFLAG(ENABLE_PROMPT_API)
|
||||
#include "third_party/blink/public/mojom/ai/ai_manager.mojom.h"
|
||||
#endif // BUILDFLAG(ENABLE_PROMPT_API)
|
||||
|
||||
namespace electron {
|
||||
|
||||
namespace {
|
||||
@@ -427,6 +431,20 @@ void UtilityProcessWrapper::OnV8FatalError(const std::string& location,
|
||||
EmitWithoutEvent("error", "FatalError", location, report);
|
||||
}
|
||||
|
||||
#if BUILDFLAG(ENABLE_PROMPT_API)
|
||||
void UtilityProcessWrapper::BindAIManager(
|
||||
std::optional<int32_t> web_contents_id,
|
||||
const url::Origin& security_origin,
|
||||
mojo::PendingReceiver<blink::mojom::AIManager> ai_manager) {
|
||||
node::mojom::BindAIManagerParamsPtr params =
|
||||
node::mojom::BindAIManagerParams::New();
|
||||
params->web_contents_id = web_contents_id;
|
||||
params->security_origin = security_origin.GetURL().spec();
|
||||
|
||||
node_service_remote_->BindAIManager(std::move(params), std::move(ai_manager));
|
||||
}
|
||||
#endif // BUILDFLAG(ENABLE_PROMPT_API)
|
||||
|
||||
// static
|
||||
raw_ptr<UtilityProcessWrapper> UtilityProcessWrapper::FromProcessId(
|
||||
base::ProcessId pid) {
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
#include "base/memory/weak_ptr.h"
|
||||
#include "base/process/process_handle.h"
|
||||
#include "content/public/browser/service_process_host.h"
|
||||
#include "electron/buildflags/buildflags.h"
|
||||
#include "mojo/public/cpp/bindings/message.h"
|
||||
#include "mojo/public/cpp/bindings/remote.h"
|
||||
#include "shell/browser/event_emitter_mixin.h"
|
||||
@@ -23,6 +24,10 @@
|
||||
#include "shell/services/node/public/mojom/node_service.mojom.h"
|
||||
#include "v8/include/v8-forward.h"
|
||||
|
||||
#if BUILDFLAG(ENABLE_PROMPT_API)
|
||||
#include "third_party/blink/public/mojom/ai/ai_manager.mojom.h"
|
||||
#endif // BUILDFLAG(ENABLE_PROMPT_API)
|
||||
|
||||
namespace gin {
|
||||
class Arguments;
|
||||
} // namespace gin
|
||||
@@ -57,6 +62,16 @@ class UtilityProcessWrapper final
|
||||
static gin_helper::Handle<UtilityProcessWrapper> Create(gin::Arguments* args);
|
||||
static raw_ptr<UtilityProcessWrapper> FromProcessId(base::ProcessId pid);
|
||||
|
||||
#if BUILDFLAG(ENABLE_PROMPT_API)
|
||||
void BindAIManager(std::optional<int32_t> web_contents_id,
|
||||
const url::Origin& security_origin,
|
||||
mojo::PendingReceiver<blink::mojom::AIManager> ai_manager);
|
||||
#endif // BUILDFLAG(ENABLE_PROMPT_API)
|
||||
|
||||
base::WeakPtr<UtilityProcessWrapper> GetWeakPtr() {
|
||||
return weak_factory_.GetWeakPtr();
|
||||
}
|
||||
|
||||
void Shutdown(uint64_t exit_code);
|
||||
|
||||
// gin_helper::Wrappable
|
||||
|
||||
@@ -223,12 +223,20 @@
|
||||
#include "shell/browser/electron_pdf_document_helper_client.h"
|
||||
#endif
|
||||
|
||||
#if BUILDFLAG(ENABLE_PROMPT_API)
|
||||
#include "shell/browser/ai/proxying_ai_manager.h"
|
||||
#endif // BUILDFLAG(ENABLE_PROMPT_API)
|
||||
|
||||
using content::BrowserThread;
|
||||
|
||||
namespace electron {
|
||||
|
||||
namespace {
|
||||
|
||||
#if BUILDFLAG(ENABLE_PROMPT_API)
|
||||
const char kAIManagerUserDataKey[] = "ai_manager";
|
||||
#endif // BUILDFLAG(ENABLE_PROMPT_API)
|
||||
|
||||
ElectronBrowserClient* g_browser_client = nullptr;
|
||||
|
||||
base::NoDestructor<std::string> g_io_thread_application_locale;
|
||||
@@ -1545,6 +1553,24 @@ void ElectronBrowserClient::
|
||||
#endif
|
||||
}
|
||||
|
||||
#if BUILDFLAG(ENABLE_PROMPT_API)
|
||||
void ElectronBrowserClient::BindAIManager(
|
||||
content::BrowserContext* browser_context,
|
||||
base::SupportsUserData* context_user_data,
|
||||
content::RenderFrameHost* rfh,
|
||||
mojo::PendingReceiver<blink::mojom::AIManager> receiver) {
|
||||
if (!context_user_data->GetUserData(kAIManagerUserDataKey)) {
|
||||
context_user_data->SetUserData(
|
||||
kAIManagerUserDataKey,
|
||||
std::make_unique<ProxyingAIManager>(browser_context, rfh));
|
||||
}
|
||||
|
||||
ProxyingAIManager* ai_manager = static_cast<ProxyingAIManager*>(
|
||||
context_user_data->GetUserData(kAIManagerUserDataKey));
|
||||
ai_manager->AddReceiver(std::move(receiver));
|
||||
}
|
||||
#endif // BUILDFLAG(ENABLE_PROMPT_API)
|
||||
|
||||
std::string ElectronBrowserClient::GetApplicationLocale() {
|
||||
return BrowserThread::CurrentlyOn(BrowserThread::IO)
|
||||
? *g_io_thread_application_locale
|
||||
|
||||
@@ -268,6 +268,14 @@ class ElectronBrowserClient : public content::ContentBrowserClient,
|
||||
const content::ServiceWorkerVersionBaseInfo& service_worker_version_info,
|
||||
blink::AssociatedInterfaceRegistry& associated_registry) override;
|
||||
|
||||
#if BUILDFLAG(ENABLE_PROMPT_API)
|
||||
void BindAIManager(
|
||||
content::BrowserContext* browser_context,
|
||||
base::SupportsUserData* context_user_data,
|
||||
content::RenderFrameHost* rfh,
|
||||
mojo::PendingReceiver<blink::mojom::AIManager> receiver) override;
|
||||
#endif // BUILDFLAG(ENABLE_PROMPT_API)
|
||||
|
||||
bool HandleExternalProtocol(
|
||||
const GURL& url,
|
||||
content::WebContents::Getter web_contents_getter,
|
||||
|
||||
@@ -8,7 +8,9 @@
|
||||
#include <vector>
|
||||
|
||||
#include "base/files/file_path.h"
|
||||
#include "base/memory/weak_ptr.h"
|
||||
#include "base/supports_user_data.h"
|
||||
#include "shell/browser/api/electron_api_utility_process.h"
|
||||
#include "shell/browser/preload_script.h"
|
||||
|
||||
namespace content {
|
||||
@@ -17,6 +19,10 @@ class BrowserContext;
|
||||
|
||||
namespace electron {
|
||||
|
||||
namespace api {
|
||||
class UtilityProcessWrapper;
|
||||
}
|
||||
|
||||
class SessionPreferences : public base::SupportsUserData::Data {
|
||||
public:
|
||||
static SessionPreferences* FromBrowserContext(
|
||||
@@ -30,6 +36,14 @@ class SessionPreferences : public base::SupportsUserData::Data {
|
||||
|
||||
bool HasServiceWorkerPreloadScript();
|
||||
|
||||
const base::WeakPtr<api::UtilityProcessWrapper>& GetLocalAIHandler() const {
|
||||
return local_ai_handler_;
|
||||
}
|
||||
|
||||
void SetLocalAIHandler(base::WeakPtr<api::UtilityProcessWrapper> handler) {
|
||||
local_ai_handler_ = handler;
|
||||
}
|
||||
|
||||
private:
|
||||
SessionPreferences();
|
||||
|
||||
@@ -37,6 +51,7 @@ class SessionPreferences : public base::SupportsUserData::Data {
|
||||
static int kLocatorKey;
|
||||
|
||||
std::vector<PreloadScript> preload_scripts_;
|
||||
base::WeakPtr<api::UtilityProcessWrapper> local_ai_handler_;
|
||||
};
|
||||
|
||||
} // namespace electron
|
||||
|
||||
@@ -55,6 +55,16 @@ v8::Local<v8::Value> CustomEmit(v8::Isolate* isolate,
|
||||
converted_args));
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
v8::Local<v8::Value> CallMethod(v8::Isolate* isolate,
|
||||
v8::Local<v8::Object> object,
|
||||
const char* method_name,
|
||||
Args&&... args) {
|
||||
v8::EscapableHandleScope scope(isolate);
|
||||
return scope.Escape(
|
||||
CustomEmit(isolate, object, method_name, std::forward<Args>(args)...));
|
||||
}
|
||||
|
||||
template <typename T, typename... Args>
|
||||
v8::Local<v8::Value> CallMethod(v8::Isolate* isolate,
|
||||
gin_helper::DeprecatedWrappable<T>* object,
|
||||
|
||||
@@ -108,6 +108,7 @@
|
||||
V(electron_browser_event_emitter) \
|
||||
V(electron_browser_system_preferences) \
|
||||
V(electron_common_net) \
|
||||
V(electron_utility_local_ai_handler) \
|
||||
V(electron_utility_parent_port)
|
||||
|
||||
#define ELECTRON_TESTING_BINDINGS(V) V(electron_common_testing)
|
||||
|
||||
@@ -142,6 +142,22 @@ node::Environment* CreateEnvironment(v8::Isolate* isolate,
|
||||
return env;
|
||||
}
|
||||
|
||||
v8::Local<v8::Object> CreateAbortController(v8::Isolate* isolate) {
|
||||
auto context = isolate->GetCurrentContext();
|
||||
auto global_object = context->Global();
|
||||
|
||||
auto value =
|
||||
global_object->Get(context, gin::StringToV8(isolate, "AbortController"))
|
||||
.ToLocalChecked();
|
||||
DCHECK(!value.IsEmpty() && value->IsObject());
|
||||
|
||||
DCHECK(value->IsFunction());
|
||||
auto constructor = value.As<v8::Function>();
|
||||
auto instance =
|
||||
constructor->NewInstance(context, 0, nullptr).ToLocalChecked();
|
||||
return instance;
|
||||
}
|
||||
|
||||
ExplicitMicrotasksScope::ExplicitMicrotasksScope(v8::MicrotaskQueue* queue)
|
||||
: microtask_queue_(queue), original_policy_(queue->microtasks_policy()) {
|
||||
// In browser-like processes, some nested run loops (macOS usually) may
|
||||
|
||||
@@ -66,6 +66,8 @@ node::Environment* CreateEnvironment(v8::Isolate* isolate,
|
||||
node::EnvironmentFlags::Flags env_flags,
|
||||
std::string_view process_type = "");
|
||||
|
||||
v8::Local<v8::Object> CreateAbortController(v8::Isolate* isolate);
|
||||
|
||||
// A scope that temporarily changes the microtask policy to explicit. Use this
|
||||
// anywhere that can trigger Node.js or uv_run().
|
||||
//
|
||||
|
||||
@@ -11,7 +11,9 @@
|
||||
#include "base/no_destructor.h"
|
||||
#include "base/process/process.h"
|
||||
#include "base/strings/utf_string_conversions.h"
|
||||
#include "electron/buildflags/buildflags.h"
|
||||
#include "electron/mas.h"
|
||||
#include "mojo/public/cpp/bindings/self_owned_receiver.h"
|
||||
#include "net/base/network_change_notifier.h"
|
||||
#include "services/network/public/cpp/wrapper_shared_url_loader_factory.h"
|
||||
#include "services/network/public/mojom/host_resolver.mojom.h"
|
||||
@@ -28,6 +30,12 @@
|
||||
#include "shell/common/crash_keys.h"
|
||||
#endif
|
||||
|
||||
#if BUILDFLAG(ENABLE_PROMPT_API)
|
||||
#include "shell/utility/ai/utility_ai_manager.h"
|
||||
#include "url/gurl.h"
|
||||
#include "url/origin.h"
|
||||
#endif // BUILDFLAG(ENABLE_PROMPT_API)
|
||||
|
||||
namespace electron {
|
||||
|
||||
mojo::Remote<node::mojom::NodeServiceClient> g_client_remote;
|
||||
@@ -193,4 +201,16 @@ void NodeService::Initialize(
|
||||
node_bindings_->StartPolling();
|
||||
}
|
||||
|
||||
#if BUILDFLAG(ENABLE_PROMPT_API)
|
||||
void NodeService::BindAIManager(
|
||||
node::mojom::BindAIManagerParamsPtr params,
|
||||
mojo::PendingReceiver<blink::mojom::AIManager> ai_manager) {
|
||||
mojo::MakeSelfOwnedReceiver(
|
||||
std::make_unique<UtilityAIManager>(
|
||||
params->web_contents_id,
|
||||
url::Origin::Create(GURL(params->security_origin))),
|
||||
std::move(ai_manager));
|
||||
}
|
||||
#endif // BUILDFLAG(ENABLE_PROMPT_API)
|
||||
|
||||
} // namespace electron
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "electron/buildflags/buildflags.h"
|
||||
#include "mojo/public/cpp/bindings/pending_receiver.h"
|
||||
#include "mojo/public/cpp/bindings/pending_remote.h"
|
||||
#include "mojo/public/cpp/bindings/receiver.h"
|
||||
@@ -66,6 +67,12 @@ class NodeService : public node::mojom::NodeService {
|
||||
mojo::PendingRemote<node::mojom::NodeServiceClient>
|
||||
client_pending_remote) override;
|
||||
|
||||
#if BUILDFLAG(ENABLE_PROMPT_API)
|
||||
void BindAIManager(
|
||||
node::mojom::BindAIManagerParamsPtr params,
|
||||
mojo::PendingReceiver<blink::mojom::AIManager> ai_manager) override;
|
||||
#endif // BUILDFLAG(ENABLE_PROMPT_API)
|
||||
|
||||
private:
|
||||
// This needs to be initialized first so that it can be destroyed last
|
||||
// after the node::Environment is destroyed. This ensures that if
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# Use of this source code is governed by the MIT license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
import("//electron/buildflags/buildflags.gni")
|
||||
import("//mojo/public/tools/bindings/mojom.gni")
|
||||
|
||||
mojom("mojom") {
|
||||
@@ -11,4 +12,8 @@ mojom("mojom") {
|
||||
"//sandbox/policy/mojom",
|
||||
"//third_party/blink/public/mojom:mojom_core",
|
||||
]
|
||||
|
||||
if (enable_prompt_api) {
|
||||
enabled_features = [ "enable_prompt_api" ]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import "mojo/public/mojom/base/file_path.mojom";
|
||||
import "sandbox/policy/mojom/sandbox.mojom";
|
||||
import "services/network/public/mojom/host_resolver.mojom";
|
||||
import "services/network/public/mojom/url_loader_factory.mojom";
|
||||
import "third_party/blink/public/mojom/ai/ai_manager.mojom";
|
||||
import "third_party/blink/public/mojom/messaging/message_port_descriptor.mojom";
|
||||
|
||||
struct NodeServiceParams {
|
||||
@@ -20,6 +21,11 @@ struct NodeServiceParams {
|
||||
bool use_network_observer_from_url_loader_factory = false;
|
||||
};
|
||||
|
||||
struct BindAIManagerParams {
|
||||
int32? web_contents_id;
|
||||
string security_origin;
|
||||
};
|
||||
|
||||
interface NodeServiceClient {
|
||||
OnV8FatalError(string location, string report);
|
||||
};
|
||||
@@ -28,4 +34,8 @@ interface NodeServiceClient {
|
||||
interface NodeService {
|
||||
Initialize(NodeServiceParams params,
|
||||
pending_remote<NodeServiceClient> client_remote);
|
||||
|
||||
[EnableIf=enable_prompt_api]
|
||||
BindAIManager(BindAIManagerParams params,
|
||||
pending_receiver<blink.mojom.AIManager> ai_manager);
|
||||
};
|
||||
|
||||
621
shell/utility/ai/utility_ai_language_model.cc
Normal file
621
shell/utility/ai/utility_ai_language_model.cc
Normal file
@@ -0,0 +1,621 @@
|
||||
// Copyright (c) 2025 Microsoft, Inc.
|
||||
// Use of this source code is governed by the MIT license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#include "shell/utility/ai/utility_ai_language_model.h"
|
||||
|
||||
#include <string_view>
|
||||
|
||||
#include "base/no_destructor.h"
|
||||
#include "base/notimplemented.h"
|
||||
#include "shell/browser/javascript_environment.h"
|
||||
#include "shell/common/gin_converters/callback_converter.h"
|
||||
#include "shell/common/gin_converters/std_converter.h"
|
||||
#include "shell/common/gin_helper/dictionary.h"
|
||||
#include "shell/common/gin_helper/event_emitter_caller.h"
|
||||
#include "shell/common/node_util.h"
|
||||
#include "shell/common/v8_util.h"
|
||||
#include "third_party/blink/public/mojom/ai/ai_common.mojom.h"
|
||||
#include "third_party/blink/public/mojom/ai/ai_language_model.mojom.h"
|
||||
#include "third_party/blink/public/mojom/ai/model_streaming_responder.mojom.h"
|
||||
|
||||
namespace gin {
|
||||
|
||||
template <>
|
||||
struct Converter<on_device_model::mojom::ResponseConstraintPtr> {
|
||||
static v8::Local<v8::Value> ToV8(
|
||||
v8::Isolate* isolate,
|
||||
const on_device_model::mojom::ResponseConstraintPtr& val) {
|
||||
if (val.is_null())
|
||||
return v8::Undefined(isolate);
|
||||
|
||||
if (val->is_json_schema()) {
|
||||
return v8::JSON::Parse(isolate->GetCurrentContext(),
|
||||
StringToV8(isolate, val->get_json_schema()))
|
||||
.ToLocalChecked();
|
||||
} else if (val->is_regex()) {
|
||||
return v8::RegExp::New(isolate->GetCurrentContext(),
|
||||
StringToV8(isolate, val->get_regex()),
|
||||
v8::RegExp::kNone)
|
||||
.ToLocalChecked();
|
||||
}
|
||||
|
||||
return v8::Undefined(isolate);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Converter<blink::mojom::AILanguageModelPromptRole> {
|
||||
static v8::Local<v8::Value> ToV8(
|
||||
v8::Isolate* isolate,
|
||||
blink::mojom::AILanguageModelPromptRole value) {
|
||||
switch (value) {
|
||||
case blink::mojom::AILanguageModelPromptRole::kSystem:
|
||||
return StringToV8(isolate, "system");
|
||||
case blink::mojom::AILanguageModelPromptRole::kUser:
|
||||
return StringToV8(isolate, "user");
|
||||
case blink::mojom::AILanguageModelPromptRole::kAssistant:
|
||||
return StringToV8(isolate, "assistant");
|
||||
default:
|
||||
return StringToV8(isolate, "unknown");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Converter<blink::mojom::AILanguageModelPromptContentPtr> {
|
||||
static v8::Local<v8::Value> ToV8(
|
||||
v8::Isolate* isolate,
|
||||
const blink::mojom::AILanguageModelPromptContentPtr& val) {
|
||||
if (val.is_null())
|
||||
return v8::Undefined(isolate);
|
||||
|
||||
auto dict = gin::Dictionary::CreateEmpty(isolate);
|
||||
|
||||
if (val->is_text()) {
|
||||
dict.Set("type", "text");
|
||||
dict.Set("text", val->get_text());
|
||||
} else if (val->is_bitmap()) {
|
||||
// Convert the bitmap to an ArrayBuffer
|
||||
// TODO - Are we going to make any guarantees about the shape of the image
|
||||
// data?
|
||||
SkBitmap& bitmap = val->get_bitmap();
|
||||
|
||||
const auto dst_info = SkImageInfo::MakeN32Premul(bitmap.dimensions());
|
||||
const size_t dst_n_bytes = dst_info.computeMinByteSize();
|
||||
auto dst_buf = v8::ArrayBuffer::New(isolate, dst_n_bytes);
|
||||
|
||||
if (!bitmap.readPixels(dst_info, dst_buf->Data(), dst_info.minRowBytes(),
|
||||
0, 0)) {
|
||||
// TODO - Handle error
|
||||
}
|
||||
|
||||
dict.Set("type", "image");
|
||||
dict.Set("image", dst_buf);
|
||||
} else if (val->is_audio()) {
|
||||
// Convert the audio data to an ArrayBuffer
|
||||
// TODO - Are we going to make any guarantees about the shape of the audio
|
||||
// data?
|
||||
on_device_model::mojom::AudioDataPtr& audio_data = val->get_audio();
|
||||
std::vector<float>& raw_data = audio_data->data;
|
||||
|
||||
const size_t dst_n_bytes =
|
||||
sizeof(std::remove_reference_t<decltype(raw_data)>::value_type) *
|
||||
raw_data.size();
|
||||
auto dst_buf = v8::ArrayBuffer::New(isolate, dst_n_bytes);
|
||||
|
||||
UNSAFE_BUFFERS(
|
||||
std::ranges::copy(raw_data, static_cast<char*>(dst_buf->Data())));
|
||||
|
||||
dict.Set("type", "audio");
|
||||
dict.Set("audio", dst_buf);
|
||||
}
|
||||
|
||||
return ConvertToV8(isolate, dict);
|
||||
}
|
||||
};
|
||||
|
||||
v8::Local<v8::Value> Converter<blink::mojom::AILanguageModelPromptPtr>::ToV8(
|
||||
v8::Isolate* isolate,
|
||||
const blink::mojom::AILanguageModelPromptPtr& val) {
|
||||
if (val.is_null())
|
||||
return v8::Undefined(isolate);
|
||||
|
||||
auto dict = gin::Dictionary::CreateEmpty(isolate);
|
||||
|
||||
dict.Set("role", val->role);
|
||||
dict.Set("content", val->content);
|
||||
dict.Set("prefix", val->is_prefix);
|
||||
|
||||
return ConvertToV8(isolate, dict);
|
||||
}
|
||||
|
||||
} // namespace gin
|
||||
|
||||
namespace electron {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr std::string_view kIsReadableStreamKey = "isReadableStream";
|
||||
constexpr std::string_view kIsLanguageModelKey = "isLanguageModel";
|
||||
constexpr std::string_view kIsLanguageModelClassKey = "isLanguageModelClass";
|
||||
|
||||
v8::Local<v8::Function> GetPrivateBoolean(v8::Isolate* const isolate,
|
||||
const v8::Local<v8::Context>& context,
|
||||
std::string_view key) {
|
||||
auto binding_key = gin::StringToV8(isolate, key);
|
||||
auto private_binding_key = v8::Private::ForApi(isolate, binding_key);
|
||||
auto global_object = context->Global();
|
||||
auto value =
|
||||
global_object->GetPrivate(context, private_binding_key).ToLocalChecked();
|
||||
if (value.IsEmpty() || !value->IsFunction()) {
|
||||
LOG(FATAL) << "Attempted to get the '" << key
|
||||
<< "' value but it was missing";
|
||||
}
|
||||
return value.As<v8::Function>();
|
||||
}
|
||||
|
||||
bool IsReadableStream(v8::Isolate* isolate, v8::Local<v8::Value> val) {
|
||||
static base::NoDestructor<v8::Global<v8::Function>> is_readable_stream;
|
||||
|
||||
auto context = isolate->GetCurrentContext();
|
||||
|
||||
if (is_readable_stream.get()->IsEmpty()) {
|
||||
is_readable_stream->Reset(
|
||||
isolate, GetPrivateBoolean(isolate, context, kIsReadableStreamKey));
|
||||
}
|
||||
|
||||
v8::Local<v8::Value> args[] = {val};
|
||||
v8::Local<v8::Value> result =
|
||||
is_readable_stream->Get(isolate)
|
||||
->Call(context, v8::Null(isolate), std::size(args), args)
|
||||
.ToLocalChecked();
|
||||
|
||||
return result->IsBoolean() && result.As<v8::Boolean>()->Value();
|
||||
}
|
||||
|
||||
// Owns itself. Will live as long as there's more data to read and
|
||||
// the Mojo remote is still connected.
|
||||
// TODO - Make this more generic to also hold a promise instead of
|
||||
// just a readable and hold an AbortController that will fire on
|
||||
// the disconnect handler stuff
|
||||
class ReadableResponder {
|
||||
public:
|
||||
ReadableResponder(v8::Isolate* isolate,
|
||||
v8::Local<v8::Object> readable_stream_reader,
|
||||
v8::Local<v8::Object> abort_controller,
|
||||
mojo::PendingRemote<blink::mojom::ModelStreamingResponder>
|
||||
pending_responder) {
|
||||
isolate_ = isolate;
|
||||
readable_stream_reader_.Reset(isolate, readable_stream_reader);
|
||||
abort_controller_.Reset(isolate, abort_controller);
|
||||
responder_.Bind(std::move(pending_responder));
|
||||
responder_.set_disconnect_handler(
|
||||
base::BindOnce(&ReadableResponder::DeleteThis, base::Unretained(this)));
|
||||
}
|
||||
|
||||
void Read(v8::Isolate* isolate) {
|
||||
v8::Local<v8::Value> val = gin_helper::CallMethod(
|
||||
isolate, readable_stream_reader_.Get(isolate), "read");
|
||||
DCHECK(val->IsPromise());
|
||||
|
||||
auto promise = val.As<v8::Promise>();
|
||||
|
||||
auto then_cb = base::BindOnce(
|
||||
[](base::WeakPtr<ReadableResponder> weak_ptr, v8::Isolate* isolate,
|
||||
v8::Local<v8::Value> result) {
|
||||
if (weak_ptr) {
|
||||
DCHECK(result->IsObject());
|
||||
|
||||
v8::Local<v8::Value> done =
|
||||
result.As<v8::Object>()
|
||||
->Get(isolate->GetCurrentContext(),
|
||||
gin::StringToV8(isolate, "done"))
|
||||
.ToLocalChecked();
|
||||
DCHECK(done->IsBoolean());
|
||||
|
||||
blink::mojom::ModelStreamingResponder* responder =
|
||||
weak_ptr->GetResponder();
|
||||
|
||||
if (done.As<v8::Boolean>()->Value()) {
|
||||
// TODO - Real token count
|
||||
responder->OnCompletion(
|
||||
blink::mojom::ModelExecutionContextInfo::New(0));
|
||||
weak_ptr->DeleteThis();
|
||||
} else {
|
||||
v8::Local<v8::Value> val =
|
||||
result.As<v8::Object>()
|
||||
->Get(isolate->GetCurrentContext(),
|
||||
gin::StringToV8(isolate, "value"))
|
||||
.ToLocalChecked();
|
||||
DCHECK(val->IsString());
|
||||
|
||||
std::string value;
|
||||
|
||||
if (gin::ConvertFromV8(isolate, val, &value)) {
|
||||
responder->OnStreaming(value);
|
||||
weak_ptr->Read(isolate);
|
||||
} else {
|
||||
// TODO - Error handling
|
||||
responder->OnError(
|
||||
blink::mojom::ModelStreamingResponseStatus::kErrorUnknown,
|
||||
/*quota_error_info=*/nullptr);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
weak_ptr_factory_.GetWeakPtr(), isolate);
|
||||
|
||||
auto catch_cb = base::BindOnce(
|
||||
[](base::WeakPtr<ReadableResponder> weak_ptr,
|
||||
v8::Local<v8::Value> result) {
|
||||
if (weak_ptr) {
|
||||
// TODO - Error is here
|
||||
// TODO - An error here is killing the utility process
|
||||
|
||||
weak_ptr->GetResponder()->OnError(
|
||||
blink::mojom::ModelStreamingResponseStatus::kErrorUnknown,
|
||||
/*quota_error_info=*/nullptr);
|
||||
}
|
||||
},
|
||||
weak_ptr_factory_.GetWeakPtr());
|
||||
|
||||
std::ignore = promise->Then(
|
||||
isolate->GetCurrentContext(),
|
||||
gin::ConvertToV8(isolate, std::move(then_cb)).As<v8::Function>(),
|
||||
gin::ConvertToV8(isolate, std::move(catch_cb)).As<v8::Function>());
|
||||
}
|
||||
|
||||
// disable copy
|
||||
ReadableResponder(const ReadableResponder&) = delete;
|
||||
ReadableResponder& operator=(const ReadableResponder&) = delete;
|
||||
|
||||
private:
|
||||
blink::mojom::ModelStreamingResponder* GetResponder() {
|
||||
return responder_.get();
|
||||
}
|
||||
|
||||
void DeleteThis() {
|
||||
LOG(ERROR) << "Deleting ReadableResponder";
|
||||
|
||||
v8::HandleScope scope{isolate_};
|
||||
gin_helper::CallMethod(isolate_, abort_controller_.Get(isolate_), "abort");
|
||||
|
||||
// TODO - Other cleanup
|
||||
delete this;
|
||||
}
|
||||
|
||||
raw_ptr<v8::Isolate> isolate_;
|
||||
v8::Global<v8::Object> readable_stream_reader_;
|
||||
v8::Global<v8::Object> abort_controller_;
|
||||
mojo::Remote<blink::mojom::ModelStreamingResponder> responder_;
|
||||
|
||||
base::WeakPtrFactory<ReadableResponder> weak_ptr_factory_{this};
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
UtilityAILanguageModel::UtilityAILanguageModel(
|
||||
v8::Local<v8::Object> language_model) {
|
||||
v8::Isolate* isolate = JavascriptEnvironment::GetIsolate();
|
||||
language_model_.Reset(isolate, language_model);
|
||||
}
|
||||
|
||||
UtilityAILanguageModel::~UtilityAILanguageModel() = default;
|
||||
|
||||
blink::mojom::ModelStreamingResponder* UtilityAILanguageModel::GetResponder(
|
||||
mojo::RemoteSetElementId responder_id) {
|
||||
return responder_set_.Get(responder_id);
|
||||
}
|
||||
|
||||
// static
|
||||
bool UtilityAILanguageModel::IsLanguageModel(v8::Isolate* isolate,
|
||||
v8::Local<v8::Value> val) {
|
||||
static base::NoDestructor<v8::Global<v8::Function>> is_language_model;
|
||||
|
||||
auto context = isolate->GetCurrentContext();
|
||||
|
||||
if (is_language_model.get()->IsEmpty()) {
|
||||
is_language_model->Reset(
|
||||
isolate, GetPrivateBoolean(isolate, context, kIsLanguageModelKey));
|
||||
}
|
||||
|
||||
v8::Local<v8::Value> args[] = {val};
|
||||
v8::Local<v8::Value> result =
|
||||
is_language_model->Get(isolate)
|
||||
->Call(context, v8::Null(isolate), std::size(args), args)
|
||||
.ToLocalChecked();
|
||||
|
||||
return result->IsBoolean() && result.As<v8::Boolean>()->Value();
|
||||
}
|
||||
|
||||
// static
|
||||
bool UtilityAILanguageModel::IsLanguageModelClass(v8::Isolate* isolate,
|
||||
v8::Local<v8::Value> val) {
|
||||
static base::NoDestructor<v8::Global<v8::Function>> is_language_model_class;
|
||||
|
||||
auto context = isolate->GetCurrentContext();
|
||||
|
||||
if (is_language_model_class.get()->IsEmpty()) {
|
||||
is_language_model_class->Reset(
|
||||
isolate, GetPrivateBoolean(isolate, context, kIsLanguageModelClassKey));
|
||||
}
|
||||
|
||||
v8::Local<v8::Value> args[] = {val};
|
||||
v8::Local<v8::Value> result =
|
||||
is_language_model_class->Get(isolate)
|
||||
->Call(context, v8::Null(isolate), std::size(args), args)
|
||||
.ToLocalChecked();
|
||||
|
||||
return result->IsBoolean() && result.As<v8::Boolean>()->Value();
|
||||
}
|
||||
|
||||
blink::mojom::AIManagerCreateLanguageModelClient*
|
||||
UtilityAILanguageModel::GetCreateLanguageModelClient(
|
||||
mojo::RemoteSetElementId responder_id) {
|
||||
return create_model_client_set_.Get(responder_id);
|
||||
}
|
||||
|
||||
void UtilityAILanguageModel::Prompt(
|
||||
std::vector<blink::mojom::AILanguageModelPromptPtr> prompts,
|
||||
on_device_model::mojom::ResponseConstraintPtr constraint,
|
||||
mojo::PendingRemote<blink::mojom::ModelStreamingResponder>
|
||||
pending_responder) {
|
||||
if (is_destroyed_) {
|
||||
mojo::Remote<blink::mojom::ModelStreamingResponder> responder(
|
||||
std::move(pending_responder));
|
||||
responder->OnError(
|
||||
blink::mojom::ModelStreamingResponseStatus::kErrorSessionDestroyed,
|
||||
/*quota_error_info=*/nullptr);
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO - Add v8::TryCatch?
|
||||
v8::Isolate* isolate = JavascriptEnvironment::GetIsolate();
|
||||
v8::HandleScope scope{isolate};
|
||||
|
||||
v8::Local<v8::Object> abort_controller = util::CreateAbortController(isolate);
|
||||
|
||||
auto options = gin_helper::Dictionary::CreateEmpty(isolate);
|
||||
if (!constraint.is_null()) {
|
||||
options.Set("responseConstraint", gin::ConvertToV8(isolate, constraint));
|
||||
}
|
||||
options.Set("signal", abort_controller
|
||||
->Get(isolate->GetCurrentContext(),
|
||||
gin::StringToV8(isolate, "signal"))
|
||||
.ToLocalChecked());
|
||||
|
||||
v8::Local<v8::Value> val = gin_helper::CallMethod(
|
||||
isolate, language_model_.Get(isolate), "prompt", prompts, options);
|
||||
|
||||
auto SendResponse =
|
||||
[](base::WeakPtr<UtilityAILanguageModel> weak_ptr, v8::Isolate* isolate,
|
||||
mojo::RemoteSetElementId responder_id, v8::Local<v8::Value> result) {
|
||||
if (weak_ptr) {
|
||||
blink::mojom::ModelStreamingResponder* responder =
|
||||
weak_ptr->GetResponder(responder_id);
|
||||
if (!responder) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::string response;
|
||||
|
||||
if (result->IsString() &&
|
||||
gin::ConvertFromV8(isolate, result, &response)) {
|
||||
responder->OnStreaming(response);
|
||||
// TODO - Pull real tokens count - need to worry about parallel
|
||||
// prompts?
|
||||
responder->OnCompletion(
|
||||
blink::mojom::ModelExecutionContextInfo::New(0));
|
||||
return;
|
||||
} else {
|
||||
// TODO - Better error handling if the developer gave us a
|
||||
// ReadableStream in the promise
|
||||
// TODO - Error handling
|
||||
responder->OnError(
|
||||
blink::mojom::ModelStreamingResponseStatus::kErrorUnknown,
|
||||
/*quota_error_info=*/nullptr);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (val->IsPromise()) {
|
||||
mojo::RemoteSetElementId responder_id =
|
||||
responder_set_.Add(std::move(pending_responder));
|
||||
|
||||
auto promise = val.As<v8::Promise>();
|
||||
|
||||
auto then_cb = base::BindOnce(SendResponse, weak_ptr_factory_.GetWeakPtr(),
|
||||
isolate, responder_id);
|
||||
|
||||
auto catch_cb = base::BindOnce(
|
||||
[](base::WeakPtr<UtilityAILanguageModel> weak_ptr,
|
||||
mojo::RemoteSetElementId responder_id, v8::Local<v8::Value> result) {
|
||||
// TODO - Error is here
|
||||
// TODO - An error here is killing the utility process
|
||||
|
||||
blink::mojom::ModelStreamingResponder* responder =
|
||||
weak_ptr->GetResponder(responder_id);
|
||||
if (!responder) {
|
||||
return;
|
||||
}
|
||||
|
||||
responder->OnError(
|
||||
blink::mojom::ModelStreamingResponseStatus::kErrorUnknown,
|
||||
/*quota_error_info=*/nullptr);
|
||||
},
|
||||
weak_ptr_factory_.GetWeakPtr(), responder_id);
|
||||
|
||||
std::ignore = promise->Then(
|
||||
isolate->GetCurrentContext(),
|
||||
gin::ConvertToV8(isolate, std::move(then_cb)).As<v8::Function>(),
|
||||
gin::ConvertToV8(isolate, std::move(catch_cb)).As<v8::Function>());
|
||||
} else if (IsReadableStream(isolate, val)) {
|
||||
v8::Local<v8::Value> reader =
|
||||
gin_helper::CallMethod(isolate, val.As<v8::Object>(), "getReader");
|
||||
DCHECK(reader->IsObject());
|
||||
|
||||
auto* readable_responder =
|
||||
new ReadableResponder(isolate, reader.As<v8::Object>(),
|
||||
abort_controller, std::move(pending_responder));
|
||||
readable_responder->Read(isolate);
|
||||
} else {
|
||||
mojo::RemoteSetElementId responder_id =
|
||||
responder_set_.Add(std::move(pending_responder));
|
||||
|
||||
SendResponse(weak_ptr_factory_.GetWeakPtr(), isolate, responder_id, val);
|
||||
}
|
||||
}
|
||||
|
||||
void UtilityAILanguageModel::Append(
|
||||
std::vector<blink::mojom::AILanguageModelPromptPtr> prompts,
|
||||
mojo::PendingRemote<blink::mojom::ModelStreamingResponder>
|
||||
pending_responder) {
|
||||
if (is_destroyed_) {
|
||||
mojo::Remote<blink::mojom::ModelStreamingResponder> responder(
|
||||
std::move(pending_responder));
|
||||
responder->OnError(
|
||||
blink::mojom::ModelStreamingResponseStatus::kErrorSessionDestroyed,
|
||||
/*quota_error_info=*/nullptr);
|
||||
return;
|
||||
}
|
||||
|
||||
mojo::RemoteSetElementId responder_id =
|
||||
responder_set_.Add(std::move(pending_responder));
|
||||
|
||||
// TODO - Add v8::TryCatch?
|
||||
v8::Isolate* isolate = JavascriptEnvironment::GetIsolate();
|
||||
v8::HandleScope scope{isolate};
|
||||
|
||||
v8::Local<v8::Value> val = gin_helper::CallMethod(
|
||||
isolate, language_model_.Get(isolate), "append", prompts);
|
||||
|
||||
auto SendResponse = [](base::WeakPtr<UtilityAILanguageModel> weak_ptr,
|
||||
v8::Isolate* isolate,
|
||||
mojo::RemoteSetElementId responder_id,
|
||||
v8::Local<v8::Value> result) {
|
||||
blink::mojom::ModelStreamingResponder* responder =
|
||||
weak_ptr->GetResponder(responder_id);
|
||||
if (!responder) {
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO - Confirm result is undefined, otherwise error for developer
|
||||
|
||||
// TODO - Pull real tokens count - need to worry about parallel
|
||||
// prompts?
|
||||
responder->OnCompletion(blink::mojom::ModelExecutionContextInfo::New(0));
|
||||
};
|
||||
|
||||
if (val->IsPromise()) {
|
||||
auto promise = val.As<v8::Promise>();
|
||||
|
||||
auto then_cb = base::BindOnce(SendResponse, weak_ptr_factory_.GetWeakPtr(),
|
||||
isolate, responder_id);
|
||||
|
||||
auto catch_cb = base::BindOnce(
|
||||
[](base::WeakPtr<UtilityAILanguageModel> weak_ptr,
|
||||
mojo::RemoteSetElementId responder_id, v8::Local<v8::Value> result) {
|
||||
// TODO - Error is here
|
||||
// TODO - An error here is killing the utility process
|
||||
|
||||
blink::mojom::ModelStreamingResponder* responder =
|
||||
weak_ptr->GetResponder(responder_id);
|
||||
if (!responder) {
|
||||
return;
|
||||
}
|
||||
|
||||
responder->OnError(
|
||||
blink::mojom::ModelStreamingResponseStatus::kErrorUnknown,
|
||||
/*quota_error_info=*/nullptr);
|
||||
},
|
||||
weak_ptr_factory_.GetWeakPtr(), responder_id);
|
||||
|
||||
std::ignore = promise->Then(
|
||||
isolate->GetCurrentContext(),
|
||||
gin::ConvertToV8(isolate, std::move(then_cb)).As<v8::Function>(),
|
||||
gin::ConvertToV8(isolate, std::move(catch_cb)).As<v8::Function>());
|
||||
} else {
|
||||
// The method is supposed to return a promise, but for
|
||||
// convenience allow developers to return a value directly
|
||||
SendResponse(weak_ptr_factory_.GetWeakPtr(), isolate, responder_id, val);
|
||||
}
|
||||
}
|
||||
|
||||
void UtilityAILanguageModel::Fork(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateLanguageModelClient>
|
||||
client) {
|
||||
// TODO - Implement this
|
||||
NOTIMPLEMENTED();
|
||||
}
|
||||
|
||||
void UtilityAILanguageModel::Destroy() {
|
||||
is_destroyed_ = true;
|
||||
|
||||
for (auto& responder : responder_set_) {
|
||||
responder->OnError(
|
||||
blink::mojom::ModelStreamingResponseStatus::kErrorSessionDestroyed,
|
||||
/*quota_error_info=*/nullptr);
|
||||
}
|
||||
responder_set_.Clear();
|
||||
|
||||
v8::Isolate* isolate = JavascriptEnvironment::GetIsolate();
|
||||
v8::HandleScope scope{isolate};
|
||||
|
||||
gin_helper::CallMethod(isolate, language_model_.Get(isolate), "destroy");
|
||||
}
|
||||
|
||||
void UtilityAILanguageModel::MeasureInputUsage(
|
||||
std::vector<blink::mojom::AILanguageModelPromptPtr> input,
|
||||
MeasureInputUsageCallback callback) {
|
||||
// TODO - Add v8::TryCatch? Otherwise an error calling the method kills
|
||||
// the utility process
|
||||
v8::Isolate* isolate = JavascriptEnvironment::GetIsolate();
|
||||
v8::HandleScope scope{isolate};
|
||||
|
||||
v8::Local<v8::Value> val = gin_helper::CallMethod(
|
||||
isolate, language_model_.Get(isolate), "measureInputUsage", input);
|
||||
|
||||
auto RunCallback = [](v8::Isolate* isolate,
|
||||
MeasureInputUsageCallback callback,
|
||||
v8::Local<v8::Value> result) {
|
||||
uint32_t input_tokens = 0;
|
||||
|
||||
if (result->IsNumber() &&
|
||||
gin::ConvertFromV8(isolate, result, &input_tokens)) {
|
||||
std::move(callback).Run(std::move(input_tokens));
|
||||
} else if (result->IsNull()) {
|
||||
std::move(callback).Run(std::nullopt);
|
||||
} else {
|
||||
// TODO - Error is here
|
||||
std::move(callback).Run(std::nullopt);
|
||||
}
|
||||
};
|
||||
|
||||
if (val->IsPromise()) {
|
||||
auto promise = val.As<v8::Promise>();
|
||||
auto split_callback = base::SplitOnceCallback(std::move(callback));
|
||||
|
||||
auto then_cb =
|
||||
base::BindOnce(RunCallback, isolate, std::move(split_callback.first));
|
||||
|
||||
auto catch_cb = base::BindOnce(
|
||||
[](MeasureInputUsageCallback callback, v8::Local<v8::Value> result) {
|
||||
// TODO - Error is here
|
||||
// TODO - Need to handle the promise rejection
|
||||
std::move(callback).Run(std::nullopt);
|
||||
},
|
||||
std::move(split_callback.second));
|
||||
|
||||
std::ignore = promise->Then(
|
||||
isolate->GetCurrentContext(),
|
||||
gin::ConvertToV8(isolate, std::move(then_cb)).As<v8::Function>(),
|
||||
gin::ConvertToV8(isolate, std::move(catch_cb)).As<v8::Function>());
|
||||
} else {
|
||||
// The method is supposed to return a promise, but for
|
||||
// convenience allow developers to return a value directly
|
||||
RunCallback(isolate, std::move(callback), val);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace electron
|
||||
76
shell/utility/ai/utility_ai_language_model.h
Normal file
76
shell/utility/ai/utility_ai_language_model.h
Normal file
@@ -0,0 +1,76 @@
|
||||
// Copyright (c) 2025 Microsoft, Inc.
|
||||
// Use of this source code is governed by the MIT license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef ELECTRON_SHELL_UTILITY_AI_UTILITY_AI_LANGUAGE_MODEL_H_
|
||||
#define ELECTRON_SHELL_UTILITY_AI_UTILITY_AI_LANGUAGE_MODEL_H_
|
||||
|
||||
#include "base/containers/flat_set.h"
|
||||
#include "base/memory/weak_ptr.h"
|
||||
#include "gin/converter.h"
|
||||
#include "mojo/public/cpp/bindings/pending_remote.h"
|
||||
#include "mojo/public/cpp/bindings/receiver.h"
|
||||
#include "mojo/public/cpp/bindings/remote_set.h"
|
||||
#include "third_party/blink/public/mojom/ai/ai_language_model.mojom.h"
|
||||
#include "v8/include/v8.h"
|
||||
|
||||
namespace electron {
|
||||
|
||||
class UtilityAILanguageModel : public blink::mojom::AILanguageModel {
|
||||
public:
|
||||
UtilityAILanguageModel(v8::Local<v8::Object> language_model);
|
||||
UtilityAILanguageModel(const UtilityAILanguageModel&) = delete;
|
||||
UtilityAILanguageModel& operator=(const UtilityAILanguageModel&) = delete;
|
||||
|
||||
~UtilityAILanguageModel() override;
|
||||
|
||||
static bool IsLanguageModel(v8::Isolate* isolate, v8::Local<v8::Value> val);
|
||||
static bool IsLanguageModelClass(v8::Isolate* isolate,
|
||||
v8::Local<v8::Value> val);
|
||||
|
||||
// `blink::mojom::AILanguageModel` implementation.
|
||||
void Prompt(std::vector<blink::mojom::AILanguageModelPromptPtr> prompts,
|
||||
on_device_model::mojom::ResponseConstraintPtr constraint,
|
||||
mojo::PendingRemote<blink::mojom::ModelStreamingResponder>
|
||||
pending_responder) override;
|
||||
void Append(std::vector<blink::mojom::AILanguageModelPromptPtr> prompts,
|
||||
mojo::PendingRemote<blink::mojom::ModelStreamingResponder>
|
||||
pending_responder) override;
|
||||
void Fork(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateLanguageModelClient>
|
||||
client) override;
|
||||
void Destroy() override;
|
||||
void MeasureInputUsage(
|
||||
std::vector<blink::mojom::AILanguageModelPromptPtr> input,
|
||||
MeasureInputUsageCallback callback) override;
|
||||
|
||||
private:
|
||||
blink::mojom::ModelStreamingResponder* GetResponder(
|
||||
mojo::RemoteSetElementId responder_id);
|
||||
blink::mojom::AIManagerCreateLanguageModelClient*
|
||||
GetCreateLanguageModelClient(mojo::RemoteSetElementId responder_id);
|
||||
|
||||
v8::Global<v8::Object> language_model_;
|
||||
bool is_destroyed_ = false;
|
||||
|
||||
mojo::RemoteSet<blink::mojom::ModelStreamingResponder> responder_set_;
|
||||
mojo::RemoteSet<blink::mojom::AIManagerCreateLanguageModelClient>
|
||||
create_model_client_set_;
|
||||
|
||||
base::WeakPtrFactory<UtilityAILanguageModel> weak_ptr_factory_{this};
|
||||
};
|
||||
|
||||
} // namespace electron
|
||||
|
||||
namespace gin {
|
||||
|
||||
template <>
|
||||
struct Converter<blink::mojom::AILanguageModelPromptPtr> {
|
||||
static v8::Local<v8::Value> ToV8(
|
||||
v8::Isolate* isolate,
|
||||
const blink::mojom::AILanguageModelPromptPtr& val);
|
||||
};
|
||||
|
||||
} // namespace gin
|
||||
|
||||
#endif // ELECTRON_SHELL_UTILITY_AI_UTILITY_AI_LANGUAGE_MODEL_H_
|
||||
553
shell/utility/ai/utility_ai_manager.cc
Normal file
553
shell/utility/ai/utility_ai_manager.cc
Normal file
@@ -0,0 +1,553 @@
|
||||
// Copyright (c) 2025 Microsoft, Inc.
|
||||
// Use of this source code is governed by the MIT license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#include "shell/utility/ai/utility_ai_manager.h"
|
||||
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
|
||||
#include "base/containers/fixed_flat_map.h"
|
||||
#include "base/notimplemented.h"
|
||||
#include "mojo/public/cpp/bindings/self_owned_receiver.h"
|
||||
#include "shell/browser/javascript_environment.h"
|
||||
#include "shell/common/gin_converters/callback_converter.h"
|
||||
#include "shell/common/gin_converters/std_converter.h"
|
||||
#include "shell/common/gin_helper/dictionary.h"
|
||||
#include "shell/common/gin_helper/event_emitter_caller.h"
|
||||
#include "shell/common/node_util.h"
|
||||
#include "shell/utility/ai/utility_ai_language_model.h"
|
||||
#include "shell/utility/api/electron_api_local_ai_handler.h"
|
||||
#include "third_party/blink/public/mojom/ai/ai_common.mojom.h"
|
||||
#include "third_party/blink/public/mojom/ai/ai_language_model.mojom.h"
|
||||
#include "url/gurl.h"
|
||||
#include "url/origin.h"
|
||||
#include "v8/include/v8.h"
|
||||
|
||||
namespace gin {
|
||||
|
||||
template <>
|
||||
struct Converter<blink::mojom::ModelAvailabilityCheckResult> {
|
||||
static bool FromV8(v8::Isolate* isolate,
|
||||
v8::Local<v8::Value> val,
|
||||
blink::mojom::ModelAvailabilityCheckResult* out) {
|
||||
using Result = blink::mojom::ModelAvailabilityCheckResult;
|
||||
static constexpr auto Lookup =
|
||||
base::MakeFixedFlatMap<std::string_view, Result>({
|
||||
{"available", Result::kAvailable},
|
||||
{"unavailable", Result::kUnavailableUnknown},
|
||||
{"downloading", Result::kDownloading},
|
||||
{"downloadable", Result::kDownloadable},
|
||||
});
|
||||
return FromV8WithLookup(isolate, val, Lookup, out);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Converter<blink::mojom::AILanguageModelParamsPtr> {
|
||||
static bool FromV8(v8::Isolate* isolate,
|
||||
v8::Local<v8::Value> val,
|
||||
blink::mojom::AILanguageModelParamsPtr* out) {
|
||||
gin_helper::Dictionary dict;
|
||||
if (!ConvertFromV8(isolate, val, &dict))
|
||||
return false;
|
||||
|
||||
auto params = blink::mojom::AILanguageModelParams::New();
|
||||
params->default_sampling_params =
|
||||
blink::mojom::AILanguageModelSamplingParams::New();
|
||||
params->max_sampling_params =
|
||||
blink::mojom::AILanguageModelSamplingParams::New();
|
||||
|
||||
if (!dict.Get("defaultTopK", &(params->default_sampling_params->top_k)))
|
||||
return false;
|
||||
|
||||
if (!dict.Get("defaultTemperature",
|
||||
&(params->default_sampling_params->temperature)))
|
||||
return false;
|
||||
|
||||
if (!dict.Get("maxTopK", &(params->max_sampling_params->top_k)))
|
||||
return false;
|
||||
|
||||
if (!dict.Get("maxTemperature",
|
||||
&(params->max_sampling_params->temperature)))
|
||||
return false;
|
||||
|
||||
*out = std::move(params);
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Converter<blink::mojom::AILanguageModelPromptType> {
|
||||
static v8::Local<v8::Value> ToV8(
|
||||
v8::Isolate* isolate,
|
||||
blink::mojom::AILanguageModelPromptType value) {
|
||||
switch (value) {
|
||||
case blink::mojom::AILanguageModelPromptType::kText:
|
||||
return StringToV8(isolate, "text");
|
||||
case blink::mojom::AILanguageModelPromptType::kImage:
|
||||
return StringToV8(isolate, "image");
|
||||
case blink::mojom::AILanguageModelPromptType::kAudio:
|
||||
return StringToV8(isolate, "audio");
|
||||
default:
|
||||
return StringToV8(isolate, "unknown");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Converter<blink::mojom::AILanguageCodePtr> {
|
||||
static v8::Local<v8::Value> ToV8(v8::Isolate* isolate,
|
||||
const blink::mojom::AILanguageCodePtr& val) {
|
||||
if (val.is_null()) {
|
||||
return v8::Undefined(isolate);
|
||||
}
|
||||
|
||||
return StringToV8(isolate, val->code);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Converter<blink::mojom::AILanguageModelExpectedPtr> {
|
||||
static v8::Local<v8::Value> ToV8(
|
||||
v8::Isolate* isolate,
|
||||
const blink::mojom::AILanguageModelExpectedPtr& val) {
|
||||
if (val.is_null()) {
|
||||
return v8::Undefined(isolate);
|
||||
}
|
||||
|
||||
auto dict = gin::Dictionary::CreateEmpty(isolate);
|
||||
|
||||
dict.Set("type", val->type);
|
||||
|
||||
if (val->languages.has_value()) {
|
||||
dict.Set("languages", val->languages.value());
|
||||
}
|
||||
|
||||
return ConvertToV8(isolate, dict);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Converter<blink::mojom::AILanguageModelCreateOptionsPtr> {
|
||||
static v8::Local<v8::Value> ToV8(
|
||||
v8::Isolate* isolate,
|
||||
const blink::mojom::AILanguageModelCreateOptionsPtr& val) {
|
||||
if (val.is_null() ||
|
||||
(val->sampling_params.is_null() && !val->expected_inputs.has_value() &&
|
||||
!val->expected_outputs.has_value() && val->initial_prompts.empty())) {
|
||||
return v8::Undefined(isolate);
|
||||
}
|
||||
|
||||
// TODO - Need to include an AbortSignal in here
|
||||
auto dict = gin::Dictionary::CreateEmpty(isolate);
|
||||
|
||||
if (!val->sampling_params.is_null()) {
|
||||
dict.Set("topK", val->sampling_params->top_k);
|
||||
dict.Set("temperature", val->sampling_params->temperature);
|
||||
}
|
||||
|
||||
if (val->expected_inputs.has_value()) {
|
||||
dict.Set("expectedInputs", val->expected_inputs.value());
|
||||
}
|
||||
|
||||
if (val->expected_outputs.has_value()) {
|
||||
dict.Set("expectedOutputs", val->expected_outputs.value());
|
||||
}
|
||||
|
||||
if (!val->initial_prompts.empty()) {
|
||||
dict.Set("initialPrompts", val->initial_prompts);
|
||||
}
|
||||
|
||||
return ConvertToV8(isolate, dict);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace gin
|
||||
|
||||
namespace electron {
|
||||
|
||||
UtilityAIManager::UtilityAIManager(std::optional<int32_t> web_contents_id,
|
||||
const url::Origin& security_origin)
|
||||
: web_contents_id_(web_contents_id), security_origin_(security_origin) {}
|
||||
|
||||
UtilityAIManager::~UtilityAIManager() = default;
|
||||
|
||||
v8::Global<v8::Object>& UtilityAIManager::GetLanguageModelClass() {
|
||||
if (language_model_class_.IsEmpty()) {
|
||||
auto& handler = electron::api::local_ai_handler::GetPromptAPIHandler();
|
||||
|
||||
if (handler.has_value()) {
|
||||
v8::Isolate* isolate = JavascriptEnvironment::GetIsolate();
|
||||
v8::HandleScope scope{isolate};
|
||||
|
||||
auto details = gin_helper::Dictionary::CreateEmpty(isolate);
|
||||
if (web_contents_id_.has_value()) {
|
||||
details.Set("webContentsId", web_contents_id_.value());
|
||||
} else {
|
||||
details.Set("webContentsId", nullptr);
|
||||
}
|
||||
details.Set("securityOrigin", security_origin_.GetURL().spec());
|
||||
|
||||
// TODO - Add v8::TryCatch?
|
||||
v8::Local<v8::Value> val = handler->Run(details);
|
||||
|
||||
// TODO - Can we validate that the class has the expected methods?
|
||||
if (!val->IsObject() ||
|
||||
!val->ToObject(isolate->GetCurrentContext())
|
||||
.ToLocalChecked()
|
||||
->IsConstructor() ||
|
||||
!UtilityAILanguageModel::IsLanguageModelClass(isolate, val)) {
|
||||
isolate->ThrowException(v8::Exception::TypeError(
|
||||
gin::StringToV8(isolate, "Must provide a constructible class")));
|
||||
} else {
|
||||
language_model_class_.Reset(
|
||||
isolate,
|
||||
val->ToObject(isolate->GetCurrentContext()).ToLocalChecked());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return language_model_class_;
|
||||
}
|
||||
|
||||
void UtilityAIManager::SendCreateLanguageModelError(
|
||||
mojo::RemoteSetElementId client_id,
|
||||
blink::mojom::AIManagerCreateClientError error) {
|
||||
blink::mojom::AIManagerCreateLanguageModelClient* client =
|
||||
create_model_client_set_.Get(client_id);
|
||||
if (!client) {
|
||||
return;
|
||||
}
|
||||
|
||||
client->OnError(error, /*quota_error_info=*/nullptr);
|
||||
}
|
||||
|
||||
void UtilityAIManager::CreateLanguageModelInternal(
|
||||
v8::Isolate* isolate,
|
||||
v8::Local<v8::Object> language_model,
|
||||
mojo::RemoteSetElementId client_id,
|
||||
blink::mojom::AILanguageModelCreateOptionsPtr options) {
|
||||
mojo::PendingRemote<blink::mojom::AILanguageModel> language_model_remote;
|
||||
|
||||
mojo::MakeSelfOwnedReceiver(
|
||||
std::make_unique<UtilityAILanguageModel>(language_model),
|
||||
language_model_remote.InitWithNewPipeAndPassReceiver());
|
||||
|
||||
gin_helper::Dictionary dict;
|
||||
uint64_t input_usage = 0;
|
||||
uint64_t input_quota = 0;
|
||||
auto model_sampling_params =
|
||||
blink::mojom::AILanguageModelSamplingParams::New();
|
||||
|
||||
if (!ConvertFromV8(isolate, language_model, &dict) ||
|
||||
!dict.Get("inputUsage", &input_usage) ||
|
||||
!dict.Get("inputQuota", &input_quota) ||
|
||||
!dict.Get("topK", &model_sampling_params->top_k) ||
|
||||
!dict.Get("temperature", &model_sampling_params->temperature)) {
|
||||
SendCreateLanguageModelError(
|
||||
client_id,
|
||||
blink::mojom::AIManagerCreateClientError::kUnableToCreateSession);
|
||||
return;
|
||||
}
|
||||
|
||||
base::flat_set<blink::mojom::AILanguageModelPromptType> enabled_input_types;
|
||||
if (options->expected_inputs.has_value()) {
|
||||
for (const auto& expected_input : options->expected_inputs.value()) {
|
||||
enabled_input_types.insert(expected_input->type);
|
||||
}
|
||||
}
|
||||
|
||||
blink::mojom::AIManagerCreateLanguageModelClient* client =
|
||||
create_model_client_set_.Get(client_id);
|
||||
if (!client) {
|
||||
return;
|
||||
}
|
||||
|
||||
client->OnResult(
|
||||
std::move(language_model_remote),
|
||||
blink::mojom::AILanguageModelInstanceInfo::New(
|
||||
input_quota, input_usage, std::move(model_sampling_params),
|
||||
std::vector<blink::mojom::AILanguageModelPromptType>(
|
||||
enabled_input_types.begin(), enabled_input_types.end())));
|
||||
}
|
||||
|
||||
void UtilityAIManager::CanCreateLanguageModel(
|
||||
blink::mojom::AILanguageModelCreateOptionsPtr options,
|
||||
CanCreateLanguageModelCallback callback) {
|
||||
v8::Global<v8::Object>& language_model_class = GetLanguageModelClass();
|
||||
blink::mojom::ModelAvailabilityCheckResult availability =
|
||||
blink::mojom::ModelAvailabilityCheckResult::kUnavailableUnknown;
|
||||
|
||||
if (language_model_class.IsEmpty()) {
|
||||
std::move(callback).Run(availability);
|
||||
} else {
|
||||
// If a handler is set, we can create a language model.
|
||||
|
||||
// TODO - Add v8::TryCatch?
|
||||
v8::Isolate* isolate = JavascriptEnvironment::GetIsolate();
|
||||
v8::HandleScope scope{isolate};
|
||||
|
||||
v8::Local<v8::Value> val = gin_helper::CallMethod(
|
||||
isolate, language_model_class.Get(isolate), "availability", options);
|
||||
|
||||
auto RunCallback = [](v8::Isolate* isolate,
|
||||
CanCreateLanguageModelCallback callback,
|
||||
v8::Local<v8::Value> result) {
|
||||
blink::mojom::ModelAvailabilityCheckResult availability =
|
||||
blink::mojom::ModelAvailabilityCheckResult::kUnavailableUnknown;
|
||||
|
||||
if (result->IsString() &&
|
||||
gin::ConvertFromV8(isolate, result, &availability)) {
|
||||
std::move(callback).Run(availability);
|
||||
} else {
|
||||
// TODO - Error is here
|
||||
std::move(callback).Run(
|
||||
blink::mojom::ModelAvailabilityCheckResult::kUnavailableUnknown);
|
||||
}
|
||||
};
|
||||
|
||||
if (val->IsPromise()) {
|
||||
auto promise = val.As<v8::Promise>();
|
||||
auto split_callback = base::SplitOnceCallback(std::move(callback));
|
||||
|
||||
auto then_cb =
|
||||
base::BindOnce(RunCallback, isolate, std::move(split_callback.first));
|
||||
|
||||
auto catch_cb = base::BindOnce(
|
||||
[](CanCreateLanguageModelCallback callback,
|
||||
v8::Local<v8::Value> result) {
|
||||
// TODO - Error is here
|
||||
// TODO - An error here is killing the utility process
|
||||
std::move(callback).Run(blink::mojom::ModelAvailabilityCheckResult::
|
||||
kUnavailableUnknown);
|
||||
},
|
||||
std::move(split_callback.second));
|
||||
|
||||
std::ignore = promise->Then(
|
||||
isolate->GetCurrentContext(),
|
||||
gin::ConvertToV8(isolate, std::move(then_cb)).As<v8::Function>(),
|
||||
gin::ConvertToV8(isolate, std::move(catch_cb)).As<v8::Function>());
|
||||
} else {
|
||||
// The method is supposed to return a promise, but for
|
||||
// convenience allow developers to return a value directly
|
||||
RunCallback(isolate, std::move(callback), val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void UtilityAIManager::CreateLanguageModel(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateLanguageModelClient>
|
||||
client,
|
||||
blink::mojom::AILanguageModelCreateOptionsPtr options) {
|
||||
v8::Global<v8::Object>& language_model_class = GetLanguageModelClass();
|
||||
|
||||
mojo::RemoteSetElementId client_id =
|
||||
create_model_client_set_.Add(std::move(client));
|
||||
|
||||
// Can't create language model if there's no language model class
|
||||
if (language_model_class.IsEmpty()) {
|
||||
SendCreateLanguageModelError(
|
||||
client_id,
|
||||
blink::mojom::AIManagerCreateClientError::kUnableToCreateSession);
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO - Add v8::TryCatch? Otherwise an error calling the method kills the
|
||||
// utility process
|
||||
v8::Isolate* isolate = JavascriptEnvironment::GetIsolate();
|
||||
v8::HandleScope scope{isolate};
|
||||
|
||||
// TODO - Store off the abort controller somewhere we can use it in disconnect
|
||||
// handler
|
||||
v8::Local<v8::Object> abort_controller = util::CreateAbortController(isolate);
|
||||
|
||||
gin_helper::Dictionary options_dict{
|
||||
isolate, gin::ConvertToV8(isolate, options).As<v8::Object>()};
|
||||
options_dict.Set("signal", abort_controller
|
||||
->Get(isolate->GetCurrentContext(),
|
||||
gin::StringToV8(isolate, "signal"))
|
||||
.ToLocalChecked());
|
||||
|
||||
v8::Local<v8::Value> val = gin_helper::CallMethod(
|
||||
isolate, language_model_class.Get(isolate), "create", options_dict);
|
||||
|
||||
if (val->IsPromise()) {
|
||||
auto promise = val.As<v8::Promise>();
|
||||
|
||||
auto then_cb = base::BindOnce(
|
||||
[](base::WeakPtr<UtilityAIManager> weak_ptr, v8::Isolate* isolate,
|
||||
mojo::RemoteSetElementId client_id,
|
||||
blink::mojom::AILanguageModelCreateOptionsPtr options,
|
||||
v8::Local<v8::Value> result) {
|
||||
if (weak_ptr) {
|
||||
blink::mojom::AILanguageModelParamsPtr params;
|
||||
if (result->IsObject() &&
|
||||
UtilityAILanguageModel::IsLanguageModel(isolate, result)) {
|
||||
weak_ptr->CreateLanguageModelInternal(
|
||||
isolate, result.As<v8::Object>(), client_id,
|
||||
std::move(options));
|
||||
} else {
|
||||
// TODO - Error is here
|
||||
weak_ptr->SendCreateLanguageModelError(
|
||||
client_id, blink::mojom::AIManagerCreateClientError::
|
||||
kUnableToCreateSession);
|
||||
}
|
||||
}
|
||||
},
|
||||
weak_ptr_factory_.GetWeakPtr(), isolate, client_id, std::move(options));
|
||||
|
||||
auto catch_cb = base::BindOnce(
|
||||
[](base::WeakPtr<UtilityAIManager> weak_ptr,
|
||||
mojo::RemoteSetElementId client_id, v8::Local<v8::Value> result) {
|
||||
// TODO - Error is here
|
||||
// TODO - Need to handle the promise rejection
|
||||
if (weak_ptr) {
|
||||
weak_ptr->SendCreateLanguageModelError(
|
||||
client_id, blink::mojom::AIManagerCreateClientError::
|
||||
kUnableToCreateSession);
|
||||
}
|
||||
},
|
||||
weak_ptr_factory_.GetWeakPtr(), client_id);
|
||||
|
||||
std::ignore = promise->Then(
|
||||
isolate->GetCurrentContext(),
|
||||
gin::ConvertToV8(isolate, std::move(then_cb)).As<v8::Function>(),
|
||||
gin::ConvertToV8(isolate, std::move(catch_cb)).As<v8::Function>());
|
||||
} else if (val->IsObject() &&
|
||||
UtilityAILanguageModel::IsLanguageModel(isolate, val)) {
|
||||
// The method is supposed to return a promise, but for
|
||||
// convenience allow developers to return a value directly
|
||||
CreateLanguageModelInternal(isolate, val.As<v8::Object>(), client_id,
|
||||
std::move(options));
|
||||
return;
|
||||
} else {
|
||||
// TODO - Error handling
|
||||
// TODO - Better error handling when the result is missing fields
|
||||
SendCreateLanguageModelError(
|
||||
client_id,
|
||||
blink::mojom::AIManagerCreateClientError::kUnableToCreateSession);
|
||||
}
|
||||
}
|
||||
|
||||
void UtilityAIManager::CanCreateSummarizer(
|
||||
blink::mojom::AISummarizerCreateOptionsPtr options,
|
||||
CanCreateSummarizerCallback callback) {
|
||||
std::move(callback).Run(
|
||||
blink::mojom::ModelAvailabilityCheckResult::kUnavailableUnknown);
|
||||
}
|
||||
|
||||
void UtilityAIManager::CreateSummarizer(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateSummarizerClient> client,
|
||||
blink::mojom::AISummarizerCreateOptionsPtr options) {
|
||||
NOTIMPLEMENTED();
|
||||
}
|
||||
|
||||
void UtilityAIManager::GetLanguageModelParams(
|
||||
GetLanguageModelParamsCallback callback) {
|
||||
v8::Global<v8::Object>& language_model_class = GetLanguageModelClass();
|
||||
|
||||
if (language_model_class.IsEmpty()) {
|
||||
std::move(callback).Run(nullptr);
|
||||
} else {
|
||||
// If a handler is set, we can get language model params
|
||||
|
||||
// TODO - Add v8::TryCatch? Otherwise an error calling the method kills
|
||||
// the utility process
|
||||
v8::Isolate* isolate = JavascriptEnvironment::GetIsolate();
|
||||
v8::HandleScope scope{isolate};
|
||||
|
||||
v8::Local<v8::Value> val = gin_helper::CallMethod(
|
||||
isolate, language_model_class.Get(isolate), "params");
|
||||
|
||||
auto RunCallback = [](v8::Isolate* isolate,
|
||||
GetLanguageModelParamsCallback callback,
|
||||
v8::Local<v8::Value> result) {
|
||||
blink::mojom::AILanguageModelParamsPtr params;
|
||||
|
||||
if (result->IsObject() && gin::ConvertFromV8(isolate, result, ¶ms)) {
|
||||
std::move(callback).Run(std::move(params));
|
||||
return;
|
||||
} else if (result->IsNull()) {
|
||||
std::move(callback).Run(nullptr);
|
||||
} else {
|
||||
// TODO - Error handling
|
||||
// TODO - Better error handling when the result is missing fields
|
||||
std::move(callback).Run(nullptr);
|
||||
}
|
||||
};
|
||||
|
||||
if (val->IsPromise()) {
|
||||
auto promise = val.As<v8::Promise>();
|
||||
auto split_callback = base::SplitOnceCallback(std::move(callback));
|
||||
|
||||
auto then_cb =
|
||||
base::BindOnce(RunCallback, isolate, std::move(split_callback.first));
|
||||
|
||||
auto catch_cb = base::BindOnce(
|
||||
[](GetLanguageModelParamsCallback callback,
|
||||
v8::Local<v8::Value> result) {
|
||||
// TODO - Error is here
|
||||
// TODO - Need to handle the promise rejection
|
||||
std::move(callback).Run(nullptr);
|
||||
},
|
||||
std::move(split_callback.second));
|
||||
|
||||
std::ignore = promise->Then(
|
||||
isolate->GetCurrentContext(),
|
||||
gin::ConvertToV8(isolate, std::move(then_cb)).As<v8::Function>(),
|
||||
gin::ConvertToV8(isolate, std::move(catch_cb)).As<v8::Function>());
|
||||
} else {
|
||||
// The method is supposed to return a promise, but for
|
||||
// convenience allow developers to return a value directly
|
||||
RunCallback(isolate, std::move(callback), val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void UtilityAIManager::CanCreateWriter(
|
||||
blink::mojom::AIWriterCreateOptionsPtr options,
|
||||
CanCreateWriterCallback callback) {
|
||||
std::move(callback).Run(
|
||||
blink::mojom::ModelAvailabilityCheckResult::kUnavailableUnknown);
|
||||
}
|
||||
|
||||
void UtilityAIManager::CreateWriter(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateWriterClient> client,
|
||||
blink::mojom::AIWriterCreateOptionsPtr options) {
|
||||
NOTIMPLEMENTED();
|
||||
}
|
||||
|
||||
void UtilityAIManager::CanCreateRewriter(
|
||||
blink::mojom::AIRewriterCreateOptionsPtr options,
|
||||
CanCreateRewriterCallback callback) {
|
||||
std::move(callback).Run(
|
||||
blink::mojom::ModelAvailabilityCheckResult::kUnavailableUnknown);
|
||||
}
|
||||
|
||||
void UtilityAIManager::CreateRewriter(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateRewriterClient> client,
|
||||
blink::mojom::AIRewriterCreateOptionsPtr options) {
|
||||
NOTIMPLEMENTED();
|
||||
}
|
||||
|
||||
void UtilityAIManager::CanCreateProofreader(
|
||||
blink::mojom::AIProofreaderCreateOptionsPtr options,
|
||||
CanCreateProofreaderCallback callback) {
|
||||
std::move(callback).Run(
|
||||
blink::mojom::ModelAvailabilityCheckResult::kUnavailableUnknown);
|
||||
}
|
||||
|
||||
void UtilityAIManager::CreateProofreader(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateProofreaderClient> client,
|
||||
blink::mojom::AIProofreaderCreateOptionsPtr options) {
|
||||
NOTIMPLEMENTED();
|
||||
}
|
||||
|
||||
void UtilityAIManager::AddModelDownloadProgressObserver(
|
||||
mojo::PendingRemote<blink::mojom::ModelDownloadProgressObserver>
|
||||
observer_remote) {
|
||||
NOTIMPLEMENTED();
|
||||
}
|
||||
|
||||
} // namespace electron
|
||||
94
shell/utility/ai/utility_ai_manager.h
Normal file
94
shell/utility/ai/utility_ai_manager.h
Normal file
@@ -0,0 +1,94 @@
|
||||
// Copyright (c) 2025 Microsoft, Inc.
|
||||
// Use of this source code is governed by the MIT license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef ELECTRON_SHELL_UTILITY_AI_UTILITY_AI_MANAGER_H_
|
||||
#define ELECTRON_SHELL_UTILITY_AI_UTILITY_AI_MANAGER_H_
|
||||
|
||||
#include <optional>
|
||||
|
||||
#include "base/memory/weak_ptr.h"
|
||||
#include "mojo/public/cpp/bindings/pending_receiver.h"
|
||||
#include "mojo/public/cpp/bindings/receiver.h"
|
||||
#include "mojo/public/cpp/bindings/remote_set.h"
|
||||
#include "third_party/blink/public/mojom/ai/ai_language_model.mojom.h"
|
||||
#include "third_party/blink/public/mojom/ai/ai_manager.mojom.h"
|
||||
#include "url/origin.h"
|
||||
#include "v8/include/v8.h"
|
||||
|
||||
namespace electron {
|
||||
|
||||
// The utility-side implementation of `blink::mojom::AIManager`.
|
||||
class UtilityAIManager : public blink::mojom::AIManager {
|
||||
public:
|
||||
UtilityAIManager(std::optional<int32_t> web_contents_id,
|
||||
const url::Origin& security_origin);
|
||||
UtilityAIManager(const UtilityAIManager&) = delete;
|
||||
UtilityAIManager& operator=(const UtilityAIManager&) = delete;
|
||||
|
||||
~UtilityAIManager() override;
|
||||
|
||||
// TODO - Create a reusable class for an in-progress LanguageModel creation
|
||||
// which properly handles the AbortController stuff
|
||||
|
||||
private:
|
||||
[[nodiscard]] v8::Global<v8::Object>& GetLanguageModelClass();
|
||||
|
||||
void SendCreateLanguageModelError(
|
||||
mojo::RemoteSetElementId client_id,
|
||||
blink::mojom::AIManagerCreateClientError error);
|
||||
|
||||
void CreateLanguageModelInternal(
|
||||
v8::Isolate* isolate,
|
||||
v8::Local<v8::Object> language_model,
|
||||
mojo::RemoteSetElementId client_id,
|
||||
blink::mojom::AILanguageModelCreateOptionsPtr options);
|
||||
|
||||
// `blink::mojom::AIManager` implementation.
|
||||
void CanCreateLanguageModel(
|
||||
blink::mojom::AILanguageModelCreateOptionsPtr options,
|
||||
CanCreateLanguageModelCallback callback) override;
|
||||
void CreateLanguageModel(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateLanguageModelClient>
|
||||
client,
|
||||
blink::mojom::AILanguageModelCreateOptionsPtr options) override;
|
||||
void CanCreateSummarizer(blink::mojom::AISummarizerCreateOptionsPtr options,
|
||||
CanCreateSummarizerCallback callback) override;
|
||||
void CreateSummarizer(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateSummarizerClient> client,
|
||||
blink::mojom::AISummarizerCreateOptionsPtr options) override;
|
||||
void GetLanguageModelParams(GetLanguageModelParamsCallback callback) override;
|
||||
void CanCreateWriter(blink::mojom::AIWriterCreateOptionsPtr options,
|
||||
CanCreateWriterCallback callback) override;
|
||||
void CreateWriter(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateWriterClient> client,
|
||||
blink::mojom::AIWriterCreateOptionsPtr options) override;
|
||||
void CanCreateRewriter(blink::mojom::AIRewriterCreateOptionsPtr options,
|
||||
CanCreateRewriterCallback callback) override;
|
||||
void CreateRewriter(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateRewriterClient> client,
|
||||
blink::mojom::AIRewriterCreateOptionsPtr options) override;
|
||||
void CanCreateProofreader(blink::mojom::AIProofreaderCreateOptionsPtr options,
|
||||
CanCreateProofreaderCallback callback) override;
|
||||
void CreateProofreader(
|
||||
mojo::PendingRemote<blink::mojom::AIManagerCreateProofreaderClient>
|
||||
client,
|
||||
blink::mojom::AIProofreaderCreateOptionsPtr options) override;
|
||||
void AddModelDownloadProgressObserver(
|
||||
mojo::PendingRemote<blink::mojom::ModelDownloadProgressObserver>
|
||||
observer_remote) override;
|
||||
|
||||
std::optional<int32_t> web_contents_id_;
|
||||
url::Origin security_origin_;
|
||||
|
||||
v8::Global<v8::Object> language_model_class_;
|
||||
|
||||
mojo::RemoteSet<blink::mojom::AIManagerCreateLanguageModelClient>
|
||||
create_model_client_set_;
|
||||
|
||||
base::WeakPtrFactory<UtilityAIManager> weak_ptr_factory_{this};
|
||||
};
|
||||
|
||||
} // namespace electron
|
||||
|
||||
#endif // ELECTRON_SHELL_UTILITY_AI_UTILITY_AI_MANAGER_H_
|
||||
53
shell/utility/api/electron_api_local_ai_handler.cc
Normal file
53
shell/utility/api/electron_api_local_ai_handler.cc
Normal file
@@ -0,0 +1,53 @@
|
||||
// Copyright (c) 2025 Microsoft, Inc.
|
||||
// Use of this source code is governed by the MIT license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#include "shell/utility/api/electron_api_local_ai_handler.h"
|
||||
|
||||
#include <optional>
|
||||
|
||||
#include "base/no_destructor.h"
|
||||
#include "shell/common/gin_converters/callback_converter.h"
|
||||
#include "shell/common/gin_helper/dictionary.h"
|
||||
#include "shell/common/node_includes.h"
|
||||
#include "v8/include/v8.h"
|
||||
|
||||
namespace electron::api::local_ai_handler {
|
||||
|
||||
void SetPromptAPIHandler(v8::Isolate* isolate, v8::Local<v8::Value> val) {
|
||||
PromptAPIHandler handler;
|
||||
if (!(val->IsNull() || gin::ConvertFromV8(isolate, val, &handler))) {
|
||||
isolate->ThrowException(v8::Exception::TypeError(
|
||||
gin::StringToV8(isolate, "Must pass null or function")));
|
||||
return;
|
||||
}
|
||||
|
||||
if (val->IsNull()) {
|
||||
GetPromptAPIHandler() = std::nullopt;
|
||||
} else {
|
||||
GetPromptAPIHandler() = handler;
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<PromptAPIHandler>& GetPromptAPIHandler() {
|
||||
static base::NoDestructor<std::optional<PromptAPIHandler>> prompt_api_handler;
|
||||
return *prompt_api_handler;
|
||||
}
|
||||
|
||||
} // namespace electron::api::local_ai_handler
|
||||
|
||||
namespace {
|
||||
|
||||
void Initialize(v8::Local<v8::Object> exports,
|
||||
v8::Local<v8::Value> unused,
|
||||
v8::Local<v8::Context> context,
|
||||
void* priv) {
|
||||
v8::Isolate* const isolate = v8::Isolate::GetCurrent();
|
||||
gin_helper::Dictionary dict{isolate, exports};
|
||||
dict.SetMethod("setPromptAPIHandler",
|
||||
&electron::api::local_ai_handler::SetPromptAPIHandler);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
NODE_LINKED_BINDING_CONTEXT_AWARE(electron_utility_local_ai_handler, Initialize)
|
||||
28
shell/utility/api/electron_api_local_ai_handler.h
Normal file
28
shell/utility/api/electron_api_local_ai_handler.h
Normal file
@@ -0,0 +1,28 @@
|
||||
// Copyright (c) 2025 Microsoft, Inc.
|
||||
// Use of this source code is governed by the MIT license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef ELECTRON_SHELL_UTILITY_API_ELECTRON_LOCAL_AI_HANDLER_H_
|
||||
#define ELECTRON_SHELL_UTILITY_API_ELECTRON_LOCAL_AI_HANDLER_H_
|
||||
|
||||
#include <optional>
|
||||
|
||||
#include "base/functional/callback_forward.h"
|
||||
#include "v8/include/v8-forward.h"
|
||||
|
||||
namespace gin_helper {
|
||||
class Dictionary;
|
||||
}
|
||||
|
||||
namespace electron::api::local_ai_handler {
|
||||
|
||||
using PromptAPIHandler =
|
||||
base::RepeatingCallback<v8::Local<v8::Value>(gin_helper::Dictionary)>;
|
||||
|
||||
void SetPromptAPIHandler(v8::Isolate* isolate, v8::Local<v8::Value> value);
|
||||
|
||||
[[nodiscard]] std::optional<PromptAPIHandler>& GetPromptAPIHandler();
|
||||
|
||||
} // namespace electron::api::local_ai_handler
|
||||
|
||||
#endif // ELECTRON_SHELL_UTILITY_API_ELECTRON_LOCAL_AI_HANDLER_H_
|
||||
3
spec/api-local-ai-handler-spec.ts
Normal file
3
spec/api-local-ai-handler-spec.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
describe('localAIHandler', () => {
|
||||
// TODO
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { app, session, BrowserWindow, net, ipcMain, Session, webFrameMain, WebFrameMain } from 'electron/main';
|
||||
import { app, session, BrowserWindow, net, ipcMain, Session, utilityProcess, webFrameMain, WebFrameMain } from 'electron/main';
|
||||
|
||||
import * as auth from 'basic-auth';
|
||||
import { expect } from 'chai';
|
||||
@@ -2029,4 +2029,35 @@ describe('session module', () => {
|
||||
expect((await cookies.get({ url: 'https://example.org/', name: 'testdotorg' })).length).to.equal(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ses.registerLocalAIHandler()', () => {
|
||||
let w: Electron.BrowserWindow;
|
||||
|
||||
beforeEach(() => {
|
||||
w = new BrowserWindow({
|
||||
show: false,
|
||||
webPreferences: {
|
||||
enableBlinkFeatures: 'AIPromptAPI'
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(closeAllWindows);
|
||||
|
||||
it('returns "unavailable" from availability() if not registered', async () => {
|
||||
await w.loadFile(path.join(fixtures, 'api', 'blank.html'));
|
||||
|
||||
expect(await w.webContents.executeJavaScript('LanguageModel.availability()')).to.equal('unavailable');
|
||||
});
|
||||
|
||||
it('returns "available" from availability() if registered', async () => {
|
||||
await w.loadFile(path.join(fixtures, 'api', 'blank.html'));
|
||||
const { session } = w.webContents;
|
||||
|
||||
const aiHandler = utilityProcess.fork(path.join(path.resolve(__dirname, 'fixtures', 'api', 'utility-process'), 'endless.js'));
|
||||
session.registerLocalAIHandler(aiHandler);
|
||||
|
||||
expect(await w.webContents.executeJavaScript('LanguageModel.availability()')).to.equal('available');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/* eslint-disable */
|
||||
|
||||
import { net, systemPreferences } from 'electron/utility';
|
||||
import { localAIHandler, net, systemPreferences, LanguageModel } from 'electron/utility';
|
||||
|
||||
process.parentPort.on('message', (e) => {
|
||||
if (e.data === 'Hello from parent!') {
|
||||
@@ -65,3 +65,36 @@ if (process.platform === 'darwin') {
|
||||
const value2 = systemPreferences.getUserDefault('Foo', 'boolean');
|
||||
console.log(value2);
|
||||
}
|
||||
|
||||
// localAIHandler
|
||||
// https://github.com/electron/electron/blob/main/docs/api/local-ai-handler.md
|
||||
|
||||
localAIHandler.setPromptAPIHandler(async (details) => {
|
||||
return class MyLanguageModel extends LanguageModel {
|
||||
private details = details;
|
||||
|
||||
static async create() {
|
||||
return new MyLanguageModel({
|
||||
inputUsage: 0,
|
||||
inputQuota: 0,
|
||||
topK: 7,
|
||||
temperature: 0
|
||||
})
|
||||
}
|
||||
|
||||
static async params () {
|
||||
return {
|
||||
defaultTopK: 12,
|
||||
defaultTemperature: 0.3,
|
||||
maxTopK: 23,
|
||||
maxTemperature: 0.5
|
||||
}
|
||||
}
|
||||
|
||||
async prompt () {
|
||||
return "Hello World"
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
localAIHandler.setPromptAPIHandler(null);
|
||||
|
||||
1
typings/internal-electron.d.ts
vendored
1
typings/internal-electron.d.ts
vendored
@@ -147,6 +147,7 @@ declare namespace Electron {
|
||||
|
||||
interface Session {
|
||||
_setDisplayMediaRequestHandler: Electron.Session['setDisplayMediaRequestHandler'];
|
||||
_registerLocalAIHandler: Electron.Session['registerLocalAIHandler'];
|
||||
}
|
||||
|
||||
type CreateWindowFunction = (options: BrowserWindowConstructorOptions) => WebContents;
|
||||
|
||||
@@ -272,10 +272,10 @@
|
||||
vscode-uri "^3.0.8"
|
||||
yaml "^2.4.5"
|
||||
|
||||
"@electron/typescript-definitions@^9.1.2":
|
||||
version "9.1.2"
|
||||
resolved "https://registry.yarnpkg.com/@electron/typescript-definitions/-/typescript-definitions-9.1.2.tgz#a9b7bfaed60a528cf1f0ce4a30f01360a27839f2"
|
||||
integrity sha512-BLxuLnvGqKUdesLXh9jB6Ll5Q4Vnb0NqJxuNY+GBz5Q8icxpW2EcHO7gIBpgX+t6sHdfRn9r6Wpwh/CKXoaJng==
|
||||
"@electron/typescript-definitions@^9.1.5":
|
||||
version "9.1.5"
|
||||
resolved "https://registry.yarnpkg.com/@electron/typescript-definitions/-/typescript-definitions-9.1.5.tgz#7a7eee8b6aef532befcc2b7d4eaf14b770e54c03"
|
||||
integrity sha512-BHLGCpy4SvOfoswRkHWTIlrhSN3z0aomSFWOFjNaEx1pvNPyC/l8Aed0LCpECDocV2PfYCv3BpYvXJGre+lRkw==
|
||||
dependencies:
|
||||
"@types/node" "^20.11.25"
|
||||
chalk "^5.3.0"
|
||||
|
||||
Reference in New Issue
Block a user