From cf70c97fd7fed01e53950f102e8fd3deefc298b3 Mon Sep 17 00:00:00 2001 From: Riccardo Date: Mon, 15 Dec 2025 13:22:19 +0100 Subject: [PATCH] add note embeddings and semantic similarity search (#1560) * First implementation of embedding feature * Improved caching * Progress on embedding index build * Added cancellation option, and updating embeds continuously * Added "Related Notes" panel, plus various tweaks * Added task deduplicator so that notes analysis command has only one running instance * Added progress/cancel features to vscode mock * Tests * refactored code into `ai` module * Added `foam.experimental.ai` feature flag, with AI features disabled by default * `waitForNoteInFoamWorkspace` now fails on timeout * Added watcher in VS Code mock, and updated related test --- packages/foam-vscode/package.json | 34 +- .../src/ai/model/embedding-cache.ts | 17 + .../src/ai/model/embeddings.test.ts | 365 +++++++++++++++++ .../foam-vscode/src/ai/model/embeddings.ts | 382 ++++++++++++++++++ .../src/ai/model/in-memory-embedding-cache.ts | 29 ++ .../providers/ollama/ollama-provider.test.ts | 294 ++++++++++++++ .../ai/providers/ollama/ollama-provider.ts | 166 ++++++++ .../src/ai/services/embedding-provider.ts | 61 +++ .../ai/services/noop-embedding-provider.ts | 27 ++ .../vscode/commands/build-embeddings.spec.ts | 88 ++++ .../ai/vscode/commands/build-embeddings.ts | 91 +++++ .../ai/vscode/commands/show-similar-notes.ts | 99 +++++ .../src/ai/vscode/panels/related-notes.ts | 138 +++++++ packages/foam-vscode/src/core/model/foam.ts | 26 +- .../foam-vscode/src/core/model/graph.test.ts | 1 - .../foam-vscode/src/core/services/progress.ts | 34 ++ .../src/core/utils/task-deduplicator.test.ts | 306 ++++++++++++++ .../src/core/utils/task-deduplicator.ts | 67 +++ packages/foam-vscode/src/extension.ts | 10 +- .../src/features/commands/index.ts | 2 + .../foam-vscode/src/features/panels/index.ts | 1 + .../foam-vscode/src/test/test-utils-vscode.ts | 4 +- packages/foam-vscode/src/test/test-utils.ts | 46 ++- packages/foam-vscode/src/test/vscode-mock.ts | 245 ++++++++++- 24 files changed, 2520 insertions(+), 13 deletions(-) create mode 100644 packages/foam-vscode/src/ai/model/embedding-cache.ts create mode 100644 packages/foam-vscode/src/ai/model/embeddings.test.ts create mode 100644 packages/foam-vscode/src/ai/model/embeddings.ts create mode 100644 packages/foam-vscode/src/ai/model/in-memory-embedding-cache.ts create mode 100644 packages/foam-vscode/src/ai/providers/ollama/ollama-provider.test.ts create mode 100644 packages/foam-vscode/src/ai/providers/ollama/ollama-provider.ts create mode 100644 packages/foam-vscode/src/ai/services/embedding-provider.ts create mode 100644 packages/foam-vscode/src/ai/services/noop-embedding-provider.ts create mode 100644 packages/foam-vscode/src/ai/vscode/commands/build-embeddings.spec.ts create mode 100644 packages/foam-vscode/src/ai/vscode/commands/build-embeddings.ts create mode 100644 packages/foam-vscode/src/ai/vscode/commands/show-similar-notes.ts create mode 100644 packages/foam-vscode/src/ai/vscode/panels/related-notes.ts create mode 100644 packages/foam-vscode/src/core/services/progress.ts create mode 100644 packages/foam-vscode/src/core/utils/task-deduplicator.test.ts create mode 100644 packages/foam-vscode/src/core/utils/task-deduplicator.ts diff --git a/packages/foam-vscode/package.json b/packages/foam-vscode/package.json index 57f10755..21274116 100644 --- a/packages/foam-vscode/package.json +++ b/packages/foam-vscode/package.json @@ -82,6 +82,13 @@ "name": "Placeholders", "icon": "$(debug-disconnect)", "contextualTitle": "Foam" + }, + { + "when": "config.foam.experimental.ai", + "id": "foam-vscode.related-notes", + "name": "Related Notes (AI)", + "icon": "$(sparkle)", + "contextualTitle": "Foam" } ] }, @@ -101,6 +108,21 @@ { "view": "foam-vscode.placeholders", "contents": "No placeholders found for selected resource or workspace." + }, + { + "view": "foam-vscode.related-notes", + "contents": "Open a note to see related notes.", + "when": "config.foam.experimental.ai && foam.relatedNotes.state == 'no-note'" + }, + { + "view": "foam-vscode.related-notes", + "contents": "Notes haven't been analyzed yet.\n[Analyze Notes](command:foam-vscode.build-embeddings)\nAnalyze your notes to discover similar content.", + "when": "config.foam.experimental.ai && foam.relatedNotes.state == 'no-embedding'" + }, + { + "view": "foam-vscode.related-notes", + "contents": "No similar notes found for the current note.", + "when": "config.foam.experimental.ai && foam.relatedNotes.state == 'ready'" } ], "menus": { @@ -384,6 +406,16 @@ "title": "Foam: Rename Tag", "icon": "$(edit)" }, + { + "command": "foam-vscode.show-similar-notes", + "title": "Foam: Show Similar Notes", + "when": "config.foam.experimental.ai" + }, + { + "command": "foam-vscode.build-embeddings", + "title": "Foam: Build Embeddings Index", + "when": "config.foam.experimental.ai" + }, { "command": "foam-vscode.views.orphans.group-by:folder", "title": "Group By Folder", @@ -694,6 +726,7 @@ "description": "Whether or not to navigate to the target daily note when a daily note snippet is selected." }, "foam.preview.embedNoteType": { + "when": "config.foam.experimental.ai", "type": "string", "default": "full-card", "enum": [ @@ -724,7 +757,6 @@ "default": false, "description": "Whether to open the graph on startup." } - } }, "keybindings": [ diff --git a/packages/foam-vscode/src/ai/model/embedding-cache.ts b/packages/foam-vscode/src/ai/model/embedding-cache.ts new file mode 100644 index 00000000..8012e19c --- /dev/null +++ b/packages/foam-vscode/src/ai/model/embedding-cache.ts @@ -0,0 +1,17 @@ +import { URI } from '../../core/model/uri'; +import { ICache } from '../../core/utils/cache'; + +type Checksum = string; + +/** + * Cache entry for embeddings + */ +export interface EmbeddingCacheEntry { + checksum: Checksum; + embedding: number[]; +} + +/** + * Cache for embeddings, keyed by URI + */ +export type EmbeddingCache = ICache; diff --git a/packages/foam-vscode/src/ai/model/embeddings.test.ts b/packages/foam-vscode/src/ai/model/embeddings.test.ts new file mode 100644 index 00000000..e7ef0bb5 --- /dev/null +++ b/packages/foam-vscode/src/ai/model/embeddings.test.ts @@ -0,0 +1,365 @@ +import { FoamEmbeddings } from './embeddings'; +import { + EmbeddingProvider, + EmbeddingProviderInfo, +} from '../services/embedding-provider'; +import { + createTestWorkspace, + InMemoryDataStore, + waitForExpect, +} from '../../test/test-utils'; +import { URI } from '../../core/model/uri'; + +// Helper to create a simple mock provider +class MockProvider implements EmbeddingProvider { + async embed(text: string): Promise { + const vector = new Array(384).fill(0); + vector[0] = text.length / 100; // Deterministic based on text length + return vector; + } + async isAvailable(): Promise { + return true; + } + getProviderInfo(): EmbeddingProviderInfo { + return { + name: 'Test Provider', + type: 'local', + model: { name: 'test-model', dimensions: 384 }, + }; + } +} + +const ROOT = [URI.parse('/', 'file')]; + +describe('FoamEmbeddings', () => { + describe('cosineSimilarity', () => { + it('should return 1 for identical vectors', () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + const vector = [1, 2, 3, 4, 5]; + const similarity = embeddings.cosineSimilarity(vector, vector); + expect(similarity).toBeCloseTo(1.0, 5); + workspace.dispose(); + }); + + it('should return 0 for orthogonal vectors', () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + const vec1 = [1, 0, 0]; + const vec2 = [0, 1, 0]; + const similarity = embeddings.cosineSimilarity(vec1, vec2); + expect(similarity).toBeCloseTo(0.0, 5); + workspace.dispose(); + }); + + it('should return -1 for opposite vectors', () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + const vec1 = [1, 0, 0]; + const vec2 = [-1, 0, 0]; + const similarity = embeddings.cosineSimilarity(vec1, vec2); + expect(similarity).toBeCloseTo(-1.0, 5); + workspace.dispose(); + }); + + it('should return 0 for zero vectors', () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + const vec1 = [0, 0, 0]; + const vec2 = [1, 2, 3]; + const similarity = embeddings.cosineSimilarity(vec1, vec2); + expect(similarity).toBe(0); + workspace.dispose(); + }); + + it('should throw error for vectors of different lengths', () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + const vec1 = [1, 2, 3]; + const vec2 = [1, 2]; + expect(() => embeddings.cosineSimilarity(vec1, vec2)).toThrow(); + workspace.dispose(); + }); + }); + + describe('updateResource', () => { + it('should create embedding for a resource', async () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + + const noteUri = URI.parse('/path/to/note.md', 'file'); + datastore.set(noteUri, '# Test Note\n\nThis is test content'); + await workspace.fetchAndSet(noteUri); + + await embeddings.updateResource(noteUri); + + const embedding = embeddings.getEmbedding(noteUri); + expect(embedding).not.toBeNull(); + expect(embedding?.length).toBe(384); + + workspace.dispose(); + }); + + it('should remove embedding when resource is deleted', async () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + + const noteUri = URI.parse('/path/to/note.md', 'file'); + datastore.set(noteUri, '# Test Note\n\nContent'); + await workspace.fetchAndSet(noteUri); + + await embeddings.updateResource(noteUri); + expect(embeddings.getEmbedding(noteUri)).not.toBeNull(); + + workspace.delete(noteUri); + await embeddings.updateResource(noteUri); + + expect(embeddings.getEmbedding(noteUri)).toBeNull(); + + workspace.dispose(); + }); + + it('should create different embeddings for different content', async () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + + const note1Uri = URI.parse('/note1.md', 'file'); + const note2Uri = URI.parse('/note2.md', 'file'); + + // Same title, different content + datastore.set(note1Uri, '# Same Title\n\nShort content'); + datastore.set( + note2Uri, + '# Same Title\n\nThis is much longer content that should produce a different embedding vector' + ); + + await workspace.fetchAndSet(note1Uri); + await workspace.fetchAndSet(note2Uri); + + await embeddings.updateResource(note1Uri); + await embeddings.updateResource(note2Uri); + + const embedding1 = embeddings.getEmbedding(note1Uri); + const embedding2 = embeddings.getEmbedding(note2Uri); + + expect(embedding1).not.toBeNull(); + expect(embedding2).not.toBeNull(); + + // Embeddings should be different because content is different + // Our mock provider uses text.length for the first vector component + expect(embedding1![0]).not.toBe(embedding2![0]); + + workspace.dispose(); + }); + }); + + describe('hasEmbeddings', () => { + it('should return false when no embeddings exist', () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + expect(embeddings.hasEmbeddings()).toBe(false); + workspace.dispose(); + }); + + it('should return true when embeddings exist', async () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + + const noteUri = URI.parse('/path/to/note.md', 'file'); + datastore.set(noteUri, '# Note\n\nContent'); + await workspace.fetchAndSet(noteUri); + + await embeddings.updateResource(noteUri); + + expect(embeddings.hasEmbeddings()).toBe(true); + + workspace.dispose(); + }); + }); + + describe('getSimilar', () => { + it('should return empty array when no embedding exists for target', () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + const uri = URI.parse('/path/to/note.md', 'file'); + + const similar = embeddings.getSimilar(uri, 5); + + expect(similar).toEqual([]); + workspace.dispose(); + }); + + it('should return similar notes sorted by similarity', async () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + + // Create notes with different content lengths + const note1Uri = URI.parse('/note1.md', 'file'); + const note2Uri = URI.parse('/note2.md', 'file'); + const note3Uri = URI.parse('/note3.md', 'file'); + + datastore.set(note1Uri, '# Note 1\n\nShort'); + datastore.set(note2Uri, '# Note 2\n\nMedium length text'); + datastore.set(note3Uri, '# Note 3\n\nVery long text content here'); + + await workspace.fetchAndSet(note1Uri); + await workspace.fetchAndSet(note2Uri); + await workspace.fetchAndSet(note3Uri); + + await embeddings.updateResource(note1Uri); + await embeddings.updateResource(note2Uri); + await embeddings.updateResource(note3Uri); + + // Get similar to note2 + const similar = embeddings.getSimilar(note2Uri, 10); + + expect(similar.length).toBe(2); // Excludes self + expect(similar[0].uri.path).toBeTruthy(); + expect(similar[0].similarity).toBeGreaterThanOrEqual( + similar[1].similarity + ); + + workspace.dispose(); + }); + + it('should respect topK parameter', async () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + + // Create multiple notes + for (let i = 0; i < 10; i++) { + const noteUri = URI.parse(`/note${i}.md`, 'file'); + datastore.set(noteUri, `# Note ${i}\n\nContent ${i}`); + await workspace.fetchAndSet(noteUri); + await embeddings.updateResource(noteUri); + } + + const target = URI.parse('/note0.md', 'file'); + const similar = embeddings.getSimilar(target, 3); + + expect(similar.length).toBe(3); + + workspace.dispose(); + }); + + it('should not include self in similar results', async () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = new FoamEmbeddings(workspace, new MockProvider()); + + const noteUri = URI.parse('/note.md', 'file'); + datastore.set(noteUri, '# Note\n\nContent'); + await workspace.fetchAndSet(noteUri); + await embeddings.updateResource(noteUri); + + const similar = embeddings.getSimilar(noteUri, 10); + + expect(similar.find(s => s.uri.path === noteUri.path)).toBeUndefined(); + + workspace.dispose(); + }); + }); + + describe('fromWorkspace with monitoring', () => { + it('should automatically update when resource is added', async () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const embeddings = FoamEmbeddings.fromWorkspace( + workspace, + new MockProvider(), + true + ); + + const noteUri = URI.parse('/new-note.md', 'file'); + datastore.set(noteUri, '# New Note\n\nContent'); + await workspace.fetchAndSet(noteUri); + + // Give it a moment to process + await new Promise(resolve => setTimeout(resolve, 100)); + + const embedding = embeddings.getEmbedding(noteUri); + expect(embedding).not.toBeNull(); + + embeddings.dispose(); + workspace.dispose(); + }); + + it('should automatically update when resource is modified', async () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const noteUri = URI.parse('/note.md', 'file'); + + datastore.set(noteUri, '# Note\n\nOriginal content'); + await workspace.fetchAndSet(noteUri); + + const embeddings = FoamEmbeddings.fromWorkspace( + workspace, + new MockProvider(), + true + ); + + await embeddings.updateResource(noteUri); + const originalEmbedding = embeddings.getEmbedding(noteUri); + + // Update the content of the note to simulate a change + datastore.set(noteUri, '# Note\n\nDifferent content that is much longer'); + + // Trigger workspace update event + await workspace.fetchAndSet(noteUri); + + // Wait for automatic update + await waitForExpect( + () => { + const newEmbedding = embeddings.getEmbedding(noteUri); + expect(newEmbedding).not.toEqual(originalEmbedding); + }, + 1000, + 50 + ); + + embeddings.dispose(); + workspace.dispose(); + }); + + it('should automatically remove embedding when resource is deleted', async () => { + const datastore = new InMemoryDataStore(); + const workspace = createTestWorkspace(ROOT, datastore); + const noteUri = URI.parse('/note.md', 'file'); + + datastore.set(noteUri, '# Note\n\nContent'); + await workspace.fetchAndSet(noteUri); + + const embeddings = FoamEmbeddings.fromWorkspace( + workspace, + new MockProvider(), + true + ); + + await embeddings.updateResource(noteUri); + expect(embeddings.getEmbedding(noteUri)).not.toBeNull(); + + workspace.delete(noteUri); + + // Give it a moment to process + await new Promise(resolve => setTimeout(resolve, 50)); + + expect(embeddings.getEmbedding(noteUri)).toBeNull(); + + embeddings.dispose(); + workspace.dispose(); + }); + }); +}); diff --git a/packages/foam-vscode/src/ai/model/embeddings.ts b/packages/foam-vscode/src/ai/model/embeddings.ts new file mode 100644 index 00000000..90c7cf33 --- /dev/null +++ b/packages/foam-vscode/src/ai/model/embeddings.ts @@ -0,0 +1,382 @@ +import { Emitter } from '../../core/common/event'; +import { IDisposable } from '../../core/common/lifecycle'; +import { Logger } from '../../core/utils/log'; +import { hash } from '../../core/utils'; +import { EmbeddingProvider, Embedding } from '../services/embedding-provider'; +import { EmbeddingCache } from './embedding-cache'; +import { + ProgressCallback, + CancellationToken, + CancellationError, +} from '../../core/services/progress'; +import { FoamWorkspace } from '../../core/model/workspace'; +import { URI } from '../../core/model/uri'; + +/** + * Represents a similar resource with its similarity score + */ +export interface SimilarResource { + uri: URI; + similarity: number; +} + +/** + * Context information for embedding progress + */ +export interface EmbeddingProgressContext { + /** URI of the current resource */ + uri: URI; + /** Title of the current resource */ + title: string; +} + +/** + * Manages embeddings for all resources in the workspace + */ +export class FoamEmbeddings implements IDisposable { + /** + * Maps resource URIs to their embeddings + */ + private embeddings: Map = new Map(); + + private onDidUpdateEmitter = new Emitter(); + onDidUpdate = this.onDidUpdateEmitter.event; + + /** + * List of disposables to destroy with the embeddings + */ + private disposables: IDisposable[] = []; + + constructor( + private readonly workspace: FoamWorkspace, + private readonly provider: EmbeddingProvider, + private readonly cache?: EmbeddingCache + ) {} + + /** + * Get the embedding for a resource + * @param uri The URI of the resource + * @returns The embedding vector, or null if not found + */ + public getEmbedding(uri: URI): number[] | null { + const embedding = this.embeddings.get(uri.path); + return embedding ? embedding.vector : null; + } + + /** + * Check if embeddings are available + * @returns true if at least one embedding exists + */ + public hasEmbeddings(): boolean { + return this.embeddings.size > 0; + } + + /** + * Get the number of embeddings + * @returns The count of embeddings + */ + public size(): number { + return this.embeddings.size; + } + + /** + * Find similar resources to a given resource + * @param uri The URI of the target resource + * @param topK The number of similar resources to return + * @returns Array of similar resources sorted by similarity (highest first) + */ + public getSimilar(uri: URI, topK: number = 10): SimilarResource[] { + const targetEmbedding = this.getEmbedding(uri); + if (!targetEmbedding) { + return []; + } + + const similarities: SimilarResource[] = []; + + for (const [path, embedding] of this.embeddings.entries()) { + // Skip self + if (path === uri.path) { + continue; + } + + const similarity = this.cosineSimilarity( + targetEmbedding, + embedding.vector + ); + similarities.push({ + uri: URI.file(path), + similarity, + }); + } + + // Sort by similarity (highest first) and take top K + similarities.sort((a, b) => b.similarity - a.similarity); + return similarities.slice(0, topK); + } + + /** + * Calculate cosine similarity between two vectors + * @param a First vector + * @param b Second vector + * @returns Similarity score between -1 and 1 (higher is more similar) + */ + public cosineSimilarity(a: number[], b: number[]): number { + if (a.length !== b.length) { + throw new Error('Vectors must have the same length'); + } + + let dotProduct = 0; + let normA = 0; + let normB = 0; + + for (let i = 0; i < a.length; i++) { + dotProduct += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + + const denominator = Math.sqrt(normA) * Math.sqrt(normB); + if (denominator === 0) { + return 0; + } + + return dotProduct / denominator; + } + + /** + * Update embeddings for a single resource + * @param uri The URI of the resource to update + * @returns The embedding vector, or null if not found/not processed + */ + public async updateResource(uri: URI): Promise { + const resource = this.workspace.find(uri); + if (!resource) { + // Resource deleted, remove embedding + this.embeddings.delete(uri.path); + if (this.cache) { + this.cache.del(uri); + } + this.onDidUpdateEmitter.fire(); + return null; + } + + // Skip non-note resources (attachments) + if (resource.type !== 'note') { + return null; + } + + try { + const content = await this.workspace.readAsMarkdown(resource.uri); + const text = this.prepareTextForEmbedding(resource.title, content); + const textChecksum = hash(text); + + // Check cache if available + if (this.cache && this.cache.has(uri)) { + const cached = this.cache.get(uri); + if (cached.checksum === textChecksum) { + Logger.debug( + `Skipping embedding for ${uri.toFsPath()} - content unchanged` + ); + // Use cached embedding + const embedding: Embedding = { + vector: cached.embedding, + createdAt: Date.now(), + }; + this.embeddings.set(uri.path, embedding); + return embedding; + } + } + + // Generate new embedding + const vector = await this.provider.embed(text); + + const embedding: Embedding = { + vector, + createdAt: Date.now(), + }; + this.embeddings.set(uri.path, embedding); + + // Update cache + if (this.cache) { + this.cache.set(uri, { + checksum: textChecksum, + embedding: vector, + }); + } + + this.onDidUpdateEmitter.fire(); + return embedding; + } catch (error) { + Logger.error(`Failed to update embedding for ${uri.toFsPath()}`, error); + return null; + } + } + + /** + * Update embeddings for all notes, processing only missing or stale ones + * @param onProgress Optional callback to report progress + * @param cancellationToken Optional token to cancel the operation + * @returns Promise that resolves when all embeddings are updated + * @throws CancellationError if the operation is cancelled + */ + public async update( + onProgress?: ProgressCallback, + cancellationToken?: CancellationToken + ): Promise { + const start = Date.now(); + + // Filter to only process notes (not attachments) + const allResources = Array.from(this.workspace.resources()); + const resources = allResources.filter(r => r.type === 'note'); + + Logger.info( + `Building embeddings for ${resources.length} notes (${allResources.length} total resources)...` + ); + + let skipped = 0; + let generated = 0; + let reused = 0; + + // Process embeddings sequentially to avoid overwhelming the service + for (let i = 0; i < resources.length; i++) { + // Check for cancellation + if (cancellationToken?.isCancellationRequested) { + Logger.info( + `Embedding build cancelled. Processed ${i}/${resources.length} notes.` + ); + throw new CancellationError('Embedding build cancelled'); + } + + const resource = resources[i]; + + onProgress?.({ + current: i + 1, + total: resources.length, + context: { + uri: resource.uri, + title: resource.title, + }, + }); + + try { + const content = await this.workspace.readAsMarkdown(resource.uri); + const text = this.prepareTextForEmbedding(resource.title, content); + const textChecksum = hash(text); + + // Check cache if available + if (this.cache && this.cache.has(resource.uri)) { + const cached = this.cache.get(resource.uri); + if (cached.checksum === textChecksum) { + // Check if we already have this embedding in memory + const existing = this.embeddings.get(resource.uri.path); + if (existing) { + // Already have current embedding, skip + reused++; + continue; + } + + // Restore from cache + this.embeddings.set(resource.uri.path, { + vector: cached.embedding, + createdAt: Date.now(), + }); + skipped++; + continue; + } + } + + // Generate new embedding + const vector = await this.provider.embed(text); + this.embeddings.set(resource.uri.path, { + vector, + createdAt: Date.now(), + }); + + // Update cache + if (this.cache) { + this.cache.set(resource.uri, { + checksum: textChecksum, + embedding: vector, + }); + } + + generated++; + } catch (error) { + Logger.error( + `Failed to generate embedding for ${resource.uri.toFsPath()}`, + error + ); + } + } + + const end = Date.now(); + Logger.info( + `Embeddings update complete: ${generated} generated, ${skipped} from cache, ${reused} already current (${ + this.embeddings.size + }/${resources.length} total) in ${end - start}ms` + ); + this.onDidUpdateEmitter.fire(); + } + + /** + * Prepare text for embedding by combining title and content + * @param title The title of the note + * @param content The markdown content of the note + * @returns The combined text to embed + */ + private prepareTextForEmbedding(title: string, content: string): string { + const parts: string[] = []; + + if (title) { + parts.push(title); + } + + if (content) { + parts.push(content); + } + + return parts.join('\n\n'); + } + + /** + * Create FoamEmbeddings from a workspace + * @param workspace The workspace to generate embeddings for + * @param provider The embedding provider to use + * @param keepMonitoring Whether to automatically update embeddings when workspace changes + * @param cache Optional cache for storing embeddings + * @returns The FoamEmbeddings instance + */ + public static fromWorkspace( + workspace: FoamWorkspace, + provider: EmbeddingProvider, + keepMonitoring: boolean = false, + cache?: EmbeddingCache + ): FoamEmbeddings { + const embeddings = new FoamEmbeddings(workspace, provider, cache); + + if (keepMonitoring) { + // Update embeddings when resources change + embeddings.disposables.push( + workspace.onDidAdd(resource => { + embeddings.updateResource(resource.uri); + }), + workspace.onDidUpdate(({ new: resource }) => { + embeddings.updateResource(resource.uri); + }), + workspace.onDidDelete(resource => { + embeddings.embeddings.delete(resource.uri.path); + embeddings.onDidUpdateEmitter.fire(); + }) + ); + } + + return embeddings; + } + + public dispose(): void { + this.onDidUpdateEmitter.dispose(); + this.disposables.forEach(d => d.dispose()); + this.disposables = []; + this.embeddings.clear(); + } +} diff --git a/packages/foam-vscode/src/ai/model/in-memory-embedding-cache.ts b/packages/foam-vscode/src/ai/model/in-memory-embedding-cache.ts new file mode 100644 index 00000000..bfc27df2 --- /dev/null +++ b/packages/foam-vscode/src/ai/model/in-memory-embedding-cache.ts @@ -0,0 +1,29 @@ +import { URI } from '../../core/model/uri'; +import { EmbeddingCache, EmbeddingCacheEntry } from './embedding-cache'; + +/** + * Simple in-memory implementation of embedding cache + */ +export class InMemoryEmbeddingCache implements EmbeddingCache { + private cache: Map = new Map(); + + get(uri: URI): EmbeddingCacheEntry { + return this.cache.get(uri.toString()); + } + + has(uri: URI): boolean { + return this.cache.has(uri.toString()); + } + + set(uri: URI, entry: EmbeddingCacheEntry): void { + this.cache.set(uri.toString(), entry); + } + + del(uri: URI): void { + this.cache.delete(uri.toString()); + } + + clear(): void { + this.cache.clear(); + } +} diff --git a/packages/foam-vscode/src/ai/providers/ollama/ollama-provider.test.ts b/packages/foam-vscode/src/ai/providers/ollama/ollama-provider.test.ts new file mode 100644 index 00000000..cf0cb521 --- /dev/null +++ b/packages/foam-vscode/src/ai/providers/ollama/ollama-provider.test.ts @@ -0,0 +1,294 @@ +import { Logger } from '../../../core/utils/log'; +import { + OllamaEmbeddingProvider, + DEFAULT_OLLAMA_CONFIG, +} from './ollama-provider'; + +Logger.setLevel('error'); + +describe('OllamaEmbeddingProvider', () => { + const originalFetch = global.fetch; + beforeEach(() => { + global.fetch = jest.fn(); + jest.clearAllMocks(); + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + global.fetch = originalFetch; + }); + + describe('constructor', () => { + it('should use default config when no config provided', () => { + const provider = new OllamaEmbeddingProvider(); + const config = provider.getConfig(); + + expect(config.url).toBe(DEFAULT_OLLAMA_CONFIG.url); + expect(config.model).toBe(DEFAULT_OLLAMA_CONFIG.model); + expect(config.timeout).toBe(DEFAULT_OLLAMA_CONFIG.timeout); + }); + + it('should merge custom config with defaults', () => { + const provider = new OllamaEmbeddingProvider({ + url: 'http://custom:11434', + }); + const config = provider.getConfig(); + + expect(config.url).toBe('http://custom:11434'); + expect(config.model).toBe(DEFAULT_OLLAMA_CONFIG.model); + }); + }); + + describe('getProviderInfo', () => { + it('should return provider information', () => { + const provider = new OllamaEmbeddingProvider(); + const info = provider.getProviderInfo(); + + expect(info.name).toBe('Ollama'); + expect(info.type).toBe('local'); + expect(info.model.name).toBe('nomic-embed-text'); + expect(info.model.dimensions).toBe(768); + expect(info.endpoint).toBe('http://localhost:11434'); + expect(info.description).toBe('Local embedding provider using Ollama'); + expect(info.metadata).toEqual({ timeout: 30000 }); + }); + + it('should return custom model name when configured', () => { + const provider = new OllamaEmbeddingProvider({ + model: 'custom-model', + }); + const info = provider.getProviderInfo(); + + expect(info.model.name).toBe('custom-model'); + }); + + it('should return custom endpoint when configured', () => { + const provider = new OllamaEmbeddingProvider({ + url: 'http://custom:8080', + }); + const info = provider.getProviderInfo(); + + expect(info.endpoint).toBe('http://custom:8080'); + }); + }); + + describe('embed', () => { + it('should successfully generate embeddings', async () => { + const mockEmbedding = new Array(768).fill(0.1); + (global.fetch as jest.Mock).mockResolvedValueOnce({ + ok: true, + json: async () => ({ embeddings: [mockEmbedding] }), + }); + + const provider = new OllamaEmbeddingProvider(); + const result = await provider.embed('test text'); + + expect(result).toEqual(mockEmbedding); + expect(global.fetch).toHaveBeenCalledWith( + 'http://localhost:11434/api/embed', + expect.objectContaining({ + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: 'nomic-embed-text', + input: ['test text'], + }), + }) + ); + }); + + it('should throw error on non-ok response', async () => { + (global.fetch as jest.Mock).mockResolvedValueOnce({ + ok: false, + status: 500, + text: async () => 'Internal server error', + }); + + const provider = new OllamaEmbeddingProvider(); + + await expect(provider.embed('test')).rejects.toThrow( + 'AI service error (500)' + ); + }); + + it('should throw error on connection refused', async () => { + (global.fetch as jest.Mock).mockRejectedValueOnce( + new Error('fetch failed') + ); + + const provider = new OllamaEmbeddingProvider(); + + await expect(provider.embed('test')).rejects.toThrow( + 'Cannot connect to Ollama' + ); + }); + + it('should timeout after configured duration', async () => { + (global.fetch as jest.Mock).mockImplementationOnce( + (_url, options) => + new Promise((_resolve, reject) => { + // Simulate abort signal being triggered + options.signal.addEventListener('abort', () => { + const error = new Error('The operation was aborted'); + error.name = 'AbortError'; + reject(error); + }); + }) + ); + + const provider = new OllamaEmbeddingProvider({ timeout: 1000 }); + const embedPromise = provider.embed('test'); + + // Fast-forward time to trigger timeout + jest.advanceTimersByTime(1001); + + await expect(embedPromise).rejects.toThrow('AI service took too long'); + }); + }); + + describe('isAvailable', () => { + it('should return true when Ollama is available', async () => { + (global.fetch as jest.Mock).mockResolvedValueOnce({ + ok: true, + }); + + const provider = new OllamaEmbeddingProvider(); + const result = await provider.isAvailable(); + + expect(result).toBe(true); + expect(global.fetch).toHaveBeenCalledWith( + 'http://localhost:11434/api/tags', + expect.objectContaining({ + method: 'GET', + }) + ); + }); + + it('should return false when Ollama is not available', async () => { + (global.fetch as jest.Mock).mockRejectedValueOnce( + new Error('Connection refused') + ); + + const provider = new OllamaEmbeddingProvider(); + const result = await provider.isAvailable(); + + expect(result).toBe(false); + }); + + it('should return false when Ollama returns non-ok status', async () => { + (global.fetch as jest.Mock).mockResolvedValueOnce({ + ok: false, + status: 404, + }); + + const provider = new OllamaEmbeddingProvider(); + const result = await provider.isAvailable(); + + expect(result).toBe(false); + }); + + it('should timeout quickly (5s) when checking availability', async () => { + (global.fetch as jest.Mock).mockImplementationOnce( + (_url, options) => + new Promise((_resolve, reject) => { + // Simulate abort signal being triggered + options.signal.addEventListener('abort', () => { + const error = new Error('The operation was aborted'); + error.name = 'AbortError'; + reject(error); + }); + }) + ); + + const provider = new OllamaEmbeddingProvider(); + const availabilityPromise = provider.isAvailable(); + + // Fast-forward time to trigger timeout (5s for availability check) + jest.advanceTimersByTime(5001); + + const result = await availabilityPromise; + expect(result).toBe(false); + }); + }); +}); + +describe('OllamaEmbeddingProvider - Integration', () => { + const provider = new OllamaEmbeddingProvider(); + + it('should handle text with unicode characters and emojis', async () => { + if (!(await provider.isAvailable())) { + console.warn('Ollama is not available, skipping test'); + return; + } + const text = 'Task completed ✔ 🚀: All systems go! 🌟'; + const embedding = await provider.embed(text); + + expect(embedding).toBeDefined(); + expect(Array.isArray(embedding)).toBe(true); + expect(embedding.length).toBe(768); // nomic-embed-text dimension + expect(embedding.every(n => typeof n === 'number')).toBe(true); + }); + + it('should handle text with various unicode characters', async () => { + if (!(await provider.isAvailable())) { + console.warn('Ollama is not available, skipping test'); + return; + } + const text = 'Hello 🌍 with émojis and spëcial çharacters • bullet ✓ check'; + const embedding = await provider.embed(text); + + expect(embedding).toBeDefined(); + expect(Array.isArray(embedding)).toBe(true); + expect(embedding.length).toBe(768); + }); + + it('should handle text with combining unicode characters', async () => { + if (!(await provider.isAvailable())) { + console.warn('Ollama is not available, skipping test'); + return; + } + // Test with combining diacriticals that could be represented differently + const text = 'café vs cafe\u0301'; // Two ways to represent é + const embedding = await provider.embed(text); + + expect(embedding).toBeDefined(); + expect(Array.isArray(embedding)).toBe(true); + expect(embedding.length).toBe(768); + }); + + it('should handle empty text', async () => { + if (!(await provider.isAvailable())) { + console.warn('Ollama is not available, skipping test'); + return; + } + const text = ''; + const embedding = await provider.embed(text); + + expect(embedding).toBeDefined(); + expect(Array.isArray(embedding)).toBe(true); + // Note: Ollama returns empty array for empty text + expect(embedding.length).toBeGreaterThanOrEqual(0); + }); + + it.each([10, 50, 60, 100, 300])( + 'should handle text of various lengths', + async length => { + if (!(await provider.isAvailable())) { + console.warn('Ollama is not available, skipping test'); + return; + } + const text = 'Lorem ipsum dolor sit amet. '.repeat(length); + try { + const embedding = await provider.embed(text); + expect(embedding).toBeDefined(); + expect(Array.isArray(embedding)).toBe(true); + expect(embedding.length).toBe(768); + } catch (error) { + throw new Error( + `Embedding failed for text of length ${text.length}: ${error}` + ); + } + } + ); +}); diff --git a/packages/foam-vscode/src/ai/providers/ollama/ollama-provider.ts b/packages/foam-vscode/src/ai/providers/ollama/ollama-provider.ts new file mode 100644 index 00000000..e7c1f4b0 --- /dev/null +++ b/packages/foam-vscode/src/ai/providers/ollama/ollama-provider.ts @@ -0,0 +1,166 @@ +import { + EmbeddingProvider, + EmbeddingProviderInfo, +} from '../../services/embedding-provider'; +import { Logger } from '../../../core/utils/log'; + +/** + * Configuration for Ollama embedding provider + */ +export interface OllamaConfig { + /** Base URL for Ollama API (default: http://localhost:11434) */ + url: string; + /** Model name to use for embeddings (default: nomic-embed-text) */ + model: string; + /** Request timeout in milliseconds (default: 30000) */ + timeout: number; +} + +/** + * Default configuration for Ollama + */ +export const DEFAULT_OLLAMA_CONFIG: OllamaConfig = { + url: 'http://localhost:11434', + model: 'nomic-embed-text', + timeout: 30000, +}; + +/** + * Ollama API response for embeddings + */ +interface OllamaEmbeddingResponse { + embeddings: number[][]; +} + +/** + * Embedding provider that uses Ollama for generating embeddings + */ +export class OllamaEmbeddingProvider implements EmbeddingProvider { + private config: OllamaConfig; + + constructor(config: Partial = {}) { + this.config = { ...DEFAULT_OLLAMA_CONFIG, ...config }; + } + + /** + * Generate an embedding for the given text + */ + async embed(text: string): Promise { + // normalize text to suitable input (format and size) + // TODO we should better handle long texts by chunking them and averaging embeddings + const input = text.substring(0, 6000).normalize(); + + try { + const controller = new AbortController(); + const timeoutId = setTimeout( + () => controller.abort(), + this.config.timeout + ); + + const response = await fetch(`${this.config.url}/api/embed`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: this.config.model, + input: [input], + }), + signal: controller.signal, + }); + + clearTimeout(timeoutId); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`AI service error (${response.status}): ${errorText}`); + } + + const data = await response.json(); + if (data.embeddings == null) { + throw new Error( + `Invalid response from AI service: ${JSON.stringify(data)}` + ); + } + return data.embeddings[0]; + } catch (error) { + if (error instanceof Error) { + if (error.name === 'AbortError') { + throw new Error( + 'AI service took too long to respond. It may be busy processing another request.' + ); + } + if ( + error.message.includes('fetch') || + error.message.includes('ECONNREFUSED') + ) { + throw new Error( + `Cannot connect to Ollama at ${this.config.url}. Make sure Ollama is installed and running.` + ); + } + } + throw error; + } + } + + /** + * Check if Ollama is available and the model is accessible + */ + async isAvailable(): Promise { + try { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 5000); + + // Try to reach the Ollama API + const response = await fetch(`${this.config.url}/api/tags`, { + method: 'GET', + signal: controller.signal, + }); + + clearTimeout(timeoutId); + + if (!response.ok) { + Logger.warn( + `Ollama API returned status ${response.status} when checking availability` + ); + return false; + } + + return true; + } catch (error) { + Logger.debug( + `Ollama not available at ${this.config.url}: ${ + error instanceof Error ? error.message : 'Unknown error' + }` + ); + return false; + } + } + + /** + * Get provider information including model details + */ + getProviderInfo(): EmbeddingProviderInfo { + return { + name: 'Ollama', + type: 'local', + model: { + name: this.config.model, + // nomic-embed-text produces 768-dimensional embeddings + dimensions: 768, + }, + description: 'Local embedding provider using Ollama', + endpoint: this.config.url, + metadata: { + timeout: this.config.timeout, + }, + }; + } + + /** + * Get current configuration + */ + getConfig(): OllamaConfig { + return { ...this.config }; + } +} diff --git a/packages/foam-vscode/src/ai/services/embedding-provider.ts b/packages/foam-vscode/src/ai/services/embedding-provider.ts new file mode 100644 index 00000000..8c55ab9f --- /dev/null +++ b/packages/foam-vscode/src/ai/services/embedding-provider.ts @@ -0,0 +1,61 @@ +/** + * Information about an embedding provider and its model + */ +export interface EmbeddingProviderInfo { + /** Human-readable name of the provider (e.g., "Ollama", "OpenAI") */ + name: string; + + /** Type of provider */ + type: 'local' | 'remote'; + + /** Model information */ + model: { + /** Model name (e.g., "nomic-embed-text", "text-embedding-3-small") */ + name: string; + /** Vector dimensions */ + dimensions: number; + }; + + /** Optional description of the provider */ + description?: string; + + /** Backend endpoint/URL if applicable */ + endpoint?: string; + + /** Additional provider-specific metadata */ + metadata?: Record; +} + +/** + * Provider interface for generating text embeddings + */ +export interface EmbeddingProvider { + /** + * Generate an embedding vector for the given text + * @param text The text to embed + * @returns A promise that resolves to the embedding vector + */ + embed(text: string): Promise; + + /** + * Check if the embedding service is available and ready to use + * @returns A promise that resolves to true if available, false otherwise + */ + isAvailable(): Promise; + + /** + * Get information about the provider and its model + * @returns Provider metadata including name, type, model info, and configuration + */ + getProviderInfo(): EmbeddingProviderInfo; +} + +/** + * Represents a text embedding with metadata + */ +export interface Embedding { + /** The embedding vector */ + vector: number[]; + /** Timestamp when the embedding was created */ + createdAt: number; +} diff --git a/packages/foam-vscode/src/ai/services/noop-embedding-provider.ts b/packages/foam-vscode/src/ai/services/noop-embedding-provider.ts new file mode 100644 index 00000000..6fd69cd1 --- /dev/null +++ b/packages/foam-vscode/src/ai/services/noop-embedding-provider.ts @@ -0,0 +1,27 @@ +import { EmbeddingProvider, EmbeddingProviderInfo } from './embedding-provider'; + +/** + * A no-op embedding provider that does nothing. + * Used when no real embedding provider is available. + */ +export class NoOpEmbeddingProvider implements EmbeddingProvider { + async embed(_text: string): Promise { + return []; + } + + async isAvailable(): Promise { + return false; + } + + getProviderInfo(): EmbeddingProviderInfo { + return { + name: 'None', + type: 'local', + model: { + name: 'none', + dimensions: 0, + }, + description: 'No embedding provider configured', + }; + } +} diff --git a/packages/foam-vscode/src/ai/vscode/commands/build-embeddings.spec.ts b/packages/foam-vscode/src/ai/vscode/commands/build-embeddings.spec.ts new file mode 100644 index 00000000..a2b822dc --- /dev/null +++ b/packages/foam-vscode/src/ai/vscode/commands/build-embeddings.spec.ts @@ -0,0 +1,88 @@ +/* @unit-ready */ +import * as vscode from 'vscode'; +import { + cleanWorkspace, + createFile, + deleteFile, + waitForNoteInFoamWorkspace, +} from '../../../test/test-utils-vscode'; +import { BUILD_EMBEDDINGS_COMMAND } from './build-embeddings'; + +describe('build-embeddings command', () => { + it('should complete successfully with no notes to analyze', async () => { + await cleanWorkspace(); + + const showInfoSpy = jest + .spyOn(vscode.window, 'showInformationMessage') + .mockResolvedValue(undefined); + + const result = await vscode.commands.executeCommand< + 'complete' | 'cancelled' | 'error' + >(BUILD_EMBEDDINGS_COMMAND.command); + + expect(result).toBe('complete'); + expect(showInfoSpy).toHaveBeenCalledWith( + expect.stringContaining('No notes found') + ); + + showInfoSpy.mockRestore(); + }); + + it('should analyze notes and report completion', async () => { + const note1 = await createFile('# Note 1\nContent here', ['note1.md']); + const note2 = await createFile('# Note 2\nMore content', ['note2.md']); + + await waitForNoteInFoamWorkspace(note1.uri); + await waitForNoteInFoamWorkspace(note2.uri); + + const showInfoSpy = jest + .spyOn(vscode.window, 'showInformationMessage') + .mockResolvedValue(undefined); + + const result = await vscode.commands.executeCommand< + 'complete' | 'cancelled' | 'error' + >(BUILD_EMBEDDINGS_COMMAND.command); + + expect(result).toBe('complete'); + expect(showInfoSpy).toHaveBeenCalledWith( + expect.stringMatching(/Analyzed.*2/) + ); + + showInfoSpy.mockRestore(); + await deleteFile(note1.uri); + await deleteFile(note2.uri); + }); + + it('should return cancelled status when operation is cancelled', async () => { + const note1 = await createFile('# Note 1\nContent', ['note1.md']); + await waitForNoteInFoamWorkspace(note1.uri); + + const tokenSource = new vscode.CancellationTokenSource(); + + const withProgressSpy = jest + .spyOn(vscode.window, 'withProgress') + .mockImplementation(async (options, task) => { + const progress = { report: () => {} }; + // Cancel immediately + tokenSource.cancel(); + return await task(progress, tokenSource.token); + }); + + const showInfoSpy = jest + .spyOn(vscode.window, 'showInformationMessage') + .mockResolvedValue(undefined); + + const result = await vscode.commands.executeCommand< + 'complete' | 'cancelled' | 'error' + >(BUILD_EMBEDDINGS_COMMAND.command); + + expect(result).toBe('cancelled'); + expect(showInfoSpy).toHaveBeenCalledWith( + expect.stringContaining('cancelled') + ); + + withProgressSpy.mockRestore(); + showInfoSpy.mockRestore(); + await deleteFile(note1.uri); + }); +}); diff --git a/packages/foam-vscode/src/ai/vscode/commands/build-embeddings.ts b/packages/foam-vscode/src/ai/vscode/commands/build-embeddings.ts new file mode 100644 index 00000000..69268150 --- /dev/null +++ b/packages/foam-vscode/src/ai/vscode/commands/build-embeddings.ts @@ -0,0 +1,91 @@ +import * as vscode from 'vscode'; +import { Foam } from '../../../core/model/foam'; +import { CancellationError } from '../../../core/services/progress'; +import { TaskDeduplicator } from '../../../core/utils/task-deduplicator'; +import { FoamWorkspace } from '../../../core/model/workspace'; +import { FoamEmbeddings } from '../../../ai/model/embeddings'; + +export const BUILD_EMBEDDINGS_COMMAND = { + command: 'foam-vscode.build-embeddings', + title: 'Foam: Analyze Notes with AI', +}; + +export default async function activate( + context: vscode.ExtensionContext, + foamPromise: Promise +) { + const foam = await foamPromise; + // Deduplicate concurrent executions + const deduplicator = new TaskDeduplicator< + 'complete' | 'cancelled' | 'error' + >(); + + context.subscriptions.push( + vscode.commands.registerCommand( + BUILD_EMBEDDINGS_COMMAND.command, + async () => { + return await deduplicator.run( + () => buildEmbeddings(foam.workspace, foam.embeddings), + () => { + vscode.window.showInformationMessage( + 'Note analysis is already in progress - waiting for it to complete' + ); + } + ); + } + ) + ); +} + +async function buildEmbeddings( + workspace: FoamWorkspace, + embeddings: FoamEmbeddings +): Promise<'complete' | 'cancelled' | 'error'> { + const notesCount = workspace.list().filter(r => r.type === 'note').length; + + if (notesCount === 0) { + vscode.window.showInformationMessage('No notes found in workspace'); + return 'complete'; + } + + // Show progress notification + return await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Window, + title: 'Analyzing notes', + cancellable: true, + }, + async (progress, token) => { + try { + await embeddings.update(progressInfo => { + const title = progressInfo.context?.title || 'Processing...'; + const increment = (1 / progressInfo.total) * 100; + progress.report({ + message: `${progressInfo.current}/${progressInfo.total} - ${title}`, + increment: increment, + }); + }, token); + + vscode.window.showInformationMessage( + `✓ Analyzed ${embeddings.size()} of ${notesCount} notes` + ); + return 'complete'; + } catch (error) { + if (error instanceof CancellationError) { + vscode.window.showInformationMessage( + 'Analysis cancelled. Run the command again to continue where you left off.' + ); + return 'cancelled'; + } + + const errorMessage = + error instanceof Error ? error.message : 'Unknown error'; + + vscode.window.showErrorMessage( + `Failed to analyze notes: ${errorMessage}` + ); + return 'error'; + } + } + ); +} diff --git a/packages/foam-vscode/src/ai/vscode/commands/show-similar-notes.ts b/packages/foam-vscode/src/ai/vscode/commands/show-similar-notes.ts new file mode 100644 index 00000000..2d90fc3d --- /dev/null +++ b/packages/foam-vscode/src/ai/vscode/commands/show-similar-notes.ts @@ -0,0 +1,99 @@ +import * as vscode from 'vscode'; +import { Foam } from '../../../core/model/foam'; +import { fromVsCodeUri, toVsCodeUri } from '../../../utils/vsc-utils'; +import { URI } from '../../../core/model/uri'; +import { BUILD_EMBEDDINGS_COMMAND } from './build-embeddings'; + +export const SHOW_SIMILAR_NOTES_COMMAND = { + command: 'foam-vscode.show-similar-notes', + title: 'Foam: Show Similar Notes', +}; + +export default async function activate( + context: vscode.ExtensionContext, + foamPromise: Promise +) { + const foam = await foamPromise; + + context.subscriptions.push( + vscode.commands.registerCommand( + SHOW_SIMILAR_NOTES_COMMAND.command, + async () => { + await showSimilarNotes(foam); + } + ) + ); +} + +async function showSimilarNotes(foam: Foam): Promise { + // Get the active editor + const editor = vscode.window.activeTextEditor; + if (!editor) { + vscode.window.showInformationMessage('Please open a note first'); + return; + } + + // Get the URI of the active document + const uri = fromVsCodeUri(editor.document.uri); + + // Check if the resource exists in workspace + const resource = foam.workspace.find(uri); + if (!resource) { + vscode.window.showInformationMessage('This file is not a note'); + return; + } + + // Ensure embeddings are up-to-date (incremental update) + const status: 'complete' | 'error' | 'cancelled' = + await vscode.commands.executeCommand(BUILD_EMBEDDINGS_COMMAND.command); + + if (status !== 'complete') { + return; + } + + // Check if embedding exists for this resource + const embedding = foam.embeddings.getEmbedding(uri); + if (!embedding) { + vscode.window.showInformationMessage( + 'This note hasn\'t been analyzed yet. Make sure the AI service is running and try the "Analyze Notes with AI" command.' + ); + return; + } + + // Get similar notes + const similar = foam.embeddings.getSimilar(uri, 10); + + if (similar.length === 0) { + vscode.window.showInformationMessage('No similar notes found'); + return; + } + + // Create quick pick items + const items: vscode.QuickPickItem[] = similar.map(item => { + const resource = foam.workspace.find(item.uri); + const title = resource?.title || item.uri.getBasename(); + const similarityPercent = (item.similarity * 100).toFixed(1); + + return { + label: `$(file) ${title}`, + description: `${similarityPercent}% similar`, + detail: item.uri.toFsPath(), + uri: item.uri, + } as vscode.QuickPickItem & { uri: URI }; + }); + + // Show quick pick + const selected = await vscode.window.showQuickPick(items, { + placeHolder: 'Select a similar note to open', + matchOnDescription: true, + matchOnDetail: true, + }); + + if (selected) { + const selectedUri = (selected as any).uri as URI; + const doc = await vscode.workspace.openTextDocument( + toVsCodeUri(selectedUri) + ); + await vscode.window.showTextDocument(doc); + } +} diff --git a/packages/foam-vscode/src/ai/vscode/panels/related-notes.ts b/packages/foam-vscode/src/ai/vscode/panels/related-notes.ts new file mode 100644 index 00000000..33084a85 --- /dev/null +++ b/packages/foam-vscode/src/ai/vscode/panels/related-notes.ts @@ -0,0 +1,138 @@ +import * as vscode from 'vscode'; +import { Foam } from '../../../core/model/foam'; +import { FoamWorkspace } from '../../../core/model/workspace'; +import { URI } from '../../../core/model/uri'; +import { fromVsCodeUri } from '../../../utils/vsc-utils'; +import { BaseTreeProvider } from '../../../features/panels/utils/base-tree-provider'; +import { ResourceTreeItem } from '../../../features/panels/utils/tree-view-utils'; +import { FoamEmbeddings } from '../../../ai/model/embeddings'; + +export default async function activate( + context: vscode.ExtensionContext, + foamPromise: Promise +) { + const foam = await foamPromise; + + const provider = new RelatedNotesTreeDataProvider( + foam.workspace, + foam.embeddings, + context.globalState + ); + + const treeView = vscode.window.createTreeView('foam-vscode.related-notes', { + treeDataProvider: provider, + showCollapseAll: false, + }); + + const updateTreeView = async () => { + const activeEditor = vscode.window.activeTextEditor; + provider.target = activeEditor + ? fromVsCodeUri(activeEditor.document.uri) + : undefined; + await provider.refresh(); + + // Update context for conditional viewsWelcome messages + vscode.commands.executeCommand( + 'setContext', + 'foam.relatedNotes.state', + provider.getState() + ); + }; + + updateTreeView(); + + context.subscriptions.push( + provider, + treeView, + foam.embeddings.onDidUpdate(() => updateTreeView()), + vscode.window.onDidChangeActiveTextEditor(() => updateTreeView()), + provider.onDidChangeTreeData(() => { + treeView.title = `Related Notes (${provider.nValues})`; + }) + ); +} + +export class RelatedNotesTreeDataProvider extends BaseTreeProvider { + public target?: URI = undefined; + public nValues = 0; + private relatedNotes: Array<{ uri: URI; similarity: number }> = []; + private currentNoteHasEmbedding = false; + + constructor( + private workspace: FoamWorkspace, + private embeddings: FoamEmbeddings, + public state: vscode.Memento + ) { + super(); + } + + async refresh(): Promise { + const uri = this.target; + + // Clear if no target or target is not a note + if (!uri) { + this.relatedNotes = []; + this.nValues = 0; + this.currentNoteHasEmbedding = false; + super.refresh(); + return; + } + + const resource = this.workspace.find(uri); + if (!resource || resource.type !== 'note') { + this.relatedNotes = []; + this.nValues = 0; + this.currentNoteHasEmbedding = false; + super.refresh(); + return; + } + + // Check if current note has an embedding + this.currentNoteHasEmbedding = this.embeddings.getEmbedding(uri) !== null; + + // Get similar notes (user can click "Build Embeddings" button if needed) + const similar = this.embeddings.getSimilar(uri, 10); + this.relatedNotes = similar.filter(n => n.similarity > 0.6); + this.nValues = this.relatedNotes.length; + super.refresh(); + } + + async getChildren(item?: vscode.TreeItem): Promise { + if (item) { + return []; + } + + // If no related notes found, show appropriate message in viewsWelcome + // The empty array will trigger the viewsWelcome content + if (this.relatedNotes.length === 0) { + return []; + } + + return this.relatedNotes + .map(({ uri, similarity }) => { + const resource = this.workspace.find(uri); + if (!resource) { + return null; + } + + const item = new ResourceTreeItem(resource, this.workspace); + // Show similarity score as percentage in description + item.description = `${Math.round(similarity * 100)}%`; + return item; + }) + .filter(item => item !== null) as ResourceTreeItem[]; + } + + /** + * Returns the current state of the related notes panel + */ + public getState(): 'no-note' | 'no-embedding' | 'ready' { + if (!this.target) { + return 'no-note'; + } + if (!this.currentNoteHasEmbedding) { + return 'no-embedding'; + } + return 'ready'; + } +} diff --git a/packages/foam-vscode/src/core/model/foam.ts b/packages/foam-vscode/src/core/model/foam.ts index 8e1566ac..45852294 100644 --- a/packages/foam-vscode/src/core/model/foam.ts +++ b/packages/foam-vscode/src/core/model/foam.ts @@ -5,6 +5,10 @@ import { FoamGraph } from './graph'; import { ResourceParser } from './note'; import { ResourceProvider } from './provider'; import { FoamTags } from './tags'; +import { FoamEmbeddings } from '../../ai/model/embeddings'; +import { InMemoryEmbeddingCache } from '../../ai/model/in-memory-embedding-cache'; +import { EmbeddingProvider } from '../../ai/services/embedding-provider'; +import { NoOpEmbeddingProvider } from '../../ai/services/noop-embedding-provider'; import { Logger, withTiming, withTimingAsync } from '../utils/log'; export interface Services { @@ -18,6 +22,7 @@ export interface Foam extends IDisposable { workspace: FoamWorkspace; graph: FoamGraph; tags: FoamTags; + embeddings: FoamEmbeddings; } export const bootstrap = async ( @@ -26,7 +31,8 @@ export const bootstrap = async ( dataStore: IDataStore, parser: ResourceParser, initialProviders: ResourceProvider[], - defaultExtension: string = '.md' + defaultExtension: string = '.md', + embeddingProvider?: EmbeddingProvider ) => { const workspace = await withTimingAsync( () => @@ -48,6 +54,22 @@ export const bootstrap = async ( ms => Logger.info(`Tags loaded in ${ms}ms`) ); + embeddingProvider = embeddingProvider ?? new NoOpEmbeddingProvider(); + const embeddings = FoamEmbeddings.fromWorkspace( + workspace, + embeddingProvider, + true, + new InMemoryEmbeddingCache() + ); + + if (await embeddingProvider.isAvailable()) { + Logger.info('Embeddings service initialized'); + } else { + Logger.warn( + 'Embedding provider not available. Semantic features will be disabled.' + ); + } + watcher?.onDidChange(async uri => { if (matcher.isMatch(uri)) { await workspace.fetchAndSet(uri); @@ -67,6 +89,7 @@ export const bootstrap = async ( workspace, graph, tags, + embeddings, services: { parser, dataStore, @@ -75,6 +98,7 @@ export const bootstrap = async ( dispose: () => { workspace.dispose(); graph.dispose(); + embeddings.dispose(); }, }; diff --git a/packages/foam-vscode/src/core/model/graph.test.ts b/packages/foam-vscode/src/core/model/graph.test.ts index 7fd9f95d..f802a489 100644 --- a/packages/foam-vscode/src/core/model/graph.test.ts +++ b/packages/foam-vscode/src/core/model/graph.test.ts @@ -146,7 +146,6 @@ describe('Graph', () => { }); const noteB = createTestNote({ uri: '/somewhere/page-b.md', - text: '## Section 1\n\n## Section 2', }); const ws = createTestWorkspace().set(noteA).set(noteB); const graph = FoamGraph.fromWorkspace(ws); diff --git a/packages/foam-vscode/src/core/services/progress.ts b/packages/foam-vscode/src/core/services/progress.ts new file mode 100644 index 00000000..73c678bf --- /dev/null +++ b/packages/foam-vscode/src/core/services/progress.ts @@ -0,0 +1,34 @@ +/** + * Generic progress information for long-running operations + */ +export interface Progress { + /** Current item being processed (1-indexed) */ + current: number; + /** Total number of items to process */ + total: number; + /** Optional context data about the current item */ + context?: T; +} + +/** + * Callback for reporting progress during operations + */ +export type ProgressCallback = (progress: Progress) => void; + +/** + * Cancellation token for aborting long-running operations + */ +export interface CancellationToken { + /** Whether cancellation has been requested */ + readonly isCancellationRequested: boolean; +} + +/** + * Exception thrown when an operation is cancelled + */ +export class CancellationError extends Error { + constructor(message: string = 'Operation cancelled') { + super(message); + this.name = 'CancellationError'; + } +} diff --git a/packages/foam-vscode/src/core/utils/task-deduplicator.test.ts b/packages/foam-vscode/src/core/utils/task-deduplicator.test.ts new file mode 100644 index 00000000..7e8adf19 --- /dev/null +++ b/packages/foam-vscode/src/core/utils/task-deduplicator.test.ts @@ -0,0 +1,306 @@ +import { TaskDeduplicator } from './task-deduplicator'; + +describe('TaskDeduplicator', () => { + describe('run', () => { + it('should execute a task and return its result', async () => { + const deduplicator = new TaskDeduplicator(); + const task = jest.fn(async () => 'result'); + + const result = await deduplicator.run(task); + + expect(result).toBe('result'); + expect(task).toHaveBeenCalledTimes(1); + }); + + it('should deduplicate concurrent calls to the same task', async () => { + const deduplicator = new TaskDeduplicator(); + let executeCount = 0; + + const task = async () => { + executeCount++; + await new Promise(resolve => setTimeout(resolve, 10)); + return 'result'; + }; + + // Start multiple concurrent calls + const [result1, result2, result3] = await Promise.all([ + deduplicator.run(task), + deduplicator.run(task), + deduplicator.run(task), + ]); + + // All should get the same result + expect(result1).toBe('result'); + expect(result2).toBe('result'); + expect(result3).toBe('result'); + + // Task should only execute once + expect(executeCount).toBe(1); + }); + + it('should call onDuplicate callback for concurrent calls', async () => { + const deduplicator = new TaskDeduplicator(); + const onDuplicate = jest.fn(); + + const task = async () => { + await new Promise(resolve => setTimeout(resolve, 10)); + return 'result'; + }; + + // Start concurrent calls + const promise1 = deduplicator.run(task); + const promise2 = deduplicator.run(task, onDuplicate); + const promise3 = deduplicator.run(task, onDuplicate); + + await Promise.all([promise1, promise2, promise3]); + + // onDuplicate should be called for the 2nd and 3rd calls + expect(onDuplicate).toHaveBeenCalledTimes(2); + }); + + it('should not call onDuplicate for the first call', async () => { + const deduplicator = new TaskDeduplicator(); + const onDuplicate = jest.fn(); + const task = jest.fn(async () => 'result'); + + await deduplicator.run(task, onDuplicate); + + expect(onDuplicate).not.toHaveBeenCalled(); + }); + + it('should allow new tasks after previous task completes', async () => { + const deduplicator = new TaskDeduplicator(); + let counter = 0; + + const task1 = async () => ++counter; + const task2 = async () => ++counter; + + const result1 = await deduplicator.run(task1); + const result2 = await deduplicator.run(task2); + + expect(result1).toBe(1); + expect(result2).toBe(2); + }); + + it('should propagate errors from the task', async () => { + const deduplicator = new TaskDeduplicator(); + const error = new Error('Task failed'); + const task = jest.fn(async () => { + throw error; + }); + + await expect(deduplicator.run(task)).rejects.toThrow('Task failed'); + }); + + it('should propagate errors to all concurrent callers', async () => { + const deduplicator = new TaskDeduplicator(); + const error = new Error('Task failed'); + + const task = async () => { + await new Promise(resolve => setTimeout(resolve, 10)); + throw error; + }; + + const promise1 = deduplicator.run(task); + const promise2 = deduplicator.run(task); + const promise3 = deduplicator.run(task); + + await expect(promise1).rejects.toThrow('Task failed'); + await expect(promise2).rejects.toThrow('Task failed'); + await expect(promise3).rejects.toThrow('Task failed'); + }); + + it('should clear running task after error', async () => { + const deduplicator = new TaskDeduplicator(); + const task1 = jest.fn(async () => { + throw new Error('Task failed'); + }); + const task2 = jest.fn(async () => 'success'); + + // First task fails + await expect(deduplicator.run(task1)).rejects.toThrow('Task failed'); + + // Second task should execute (not deduplicated) + const result = await deduplicator.run(task2); + + expect(result).toBe('success'); + expect(task1).toHaveBeenCalledTimes(1); + expect(task2).toHaveBeenCalledTimes(1); + }); + + it('should handle different return types', async () => { + // String + const stringDeduplicator = new TaskDeduplicator(); + const stringResult = await stringDeduplicator.run(async () => 'test'); + expect(stringResult).toBe('test'); + + // Number + const numberDeduplicator = new TaskDeduplicator(); + const numberResult = await numberDeduplicator.run(async () => 42); + expect(numberResult).toBe(42); + + // Object + const objectDeduplicator = new TaskDeduplicator<{ value: string }>(); + const objectResult = await objectDeduplicator.run(async () => ({ + value: 'test', + })); + expect(objectResult).toEqual({ value: 'test' }); + + // Union types + type Status = 'complete' | 'cancelled' | 'error'; + const statusDeduplicator = new TaskDeduplicator(); + const statusResult = await statusDeduplicator.run( + async () => 'complete' as Status + ); + expect(statusResult).toBe('complete'); + }); + }); + + describe('isRunning', () => { + it('should return false when no task is running', () => { + const deduplicator = new TaskDeduplicator(); + + expect(deduplicator.isRunning()).toBe(false); + }); + + it('should return true when a task is running', async () => { + const deduplicator = new TaskDeduplicator(); + + const task = async () => { + await new Promise(resolve => setTimeout(resolve, 10)); + return 'result'; + }; + + const promise = deduplicator.run(task); + + expect(deduplicator.isRunning()).toBe(true); + + await promise; + }); + + it('should return false after task completes', async () => { + const deduplicator = new TaskDeduplicator(); + const task = jest.fn(async () => 'result'); + + await deduplicator.run(task); + + expect(deduplicator.isRunning()).toBe(false); + }); + + it('should return false after task fails', async () => { + const deduplicator = new TaskDeduplicator(); + const task = jest.fn(async () => { + throw new Error('Failed'); + }); + + await expect(deduplicator.run(task)).rejects.toThrow('Failed'); + + expect(deduplicator.isRunning()).toBe(false); + }); + }); + + describe('clear', () => { + it('should clear the running task reference', async () => { + const deduplicator = new TaskDeduplicator(); + + const task = async () => { + await new Promise(resolve => setTimeout(resolve, 10)); + return 'result'; + }; + + const promise = deduplicator.run(task); + + expect(deduplicator.isRunning()).toBe(true); + + deduplicator.clear(); + + expect(deduplicator.isRunning()).toBe(false); + + // Original promise should still complete + await expect(promise).resolves.toBe('result'); + }); + + it('should allow new task after manual clear', async () => { + const deduplicator = new TaskDeduplicator(); + let executeCount = 0; + + const task = async () => { + executeCount++; + await new Promise(resolve => setTimeout(resolve, 50)); + return 'result'; + }; + + // Start first task + const promise1 = deduplicator.run(task); + + // Clear while still running + deduplicator.clear(); + + // Start second task (should not be deduplicated) + const promise2 = deduplicator.run(task); + + await Promise.all([promise1, promise2]); + + // Both tasks should have executed + expect(executeCount).toBe(2); + }); + + it('should be safe to call when no task is running', () => { + const deduplicator = new TaskDeduplicator(); + + expect(() => deduplicator.clear()).not.toThrow(); + expect(deduplicator.isRunning()).toBe(false); + }); + }); + + describe('edge cases', () => { + it('should handle tasks that resolve immediately', async () => { + const deduplicator = new TaskDeduplicator(); + const task = jest.fn(async () => 'immediate'); + + const result = await deduplicator.run(task); + + expect(result).toBe('immediate'); + expect(deduplicator.isRunning()).toBe(false); + }); + + it('should handle tasks that throw synchronously', async () => { + const deduplicator = new TaskDeduplicator(); + const task = jest.fn(() => { + throw new Error('Sync error'); + }); + + await expect(deduplicator.run(task as any)).rejects.toThrow('Sync error'); + expect(deduplicator.isRunning()).toBe(false); + }); + + it('should handle null/undefined results', async () => { + const nullDeduplicator = new TaskDeduplicator(); + const nullResult = await nullDeduplicator.run(async () => null); + expect(nullResult).toBeNull(); + + const undefinedDeduplicator = new TaskDeduplicator(); + const undefinedResult = await undefinedDeduplicator.run( + async () => undefined + ); + expect(undefinedResult).toBeUndefined(); + }); + + it('should handle sequential calls with delays between them', async () => { + const deduplicator = new TaskDeduplicator(); + let counter = 0; + + const task = async () => { + await new Promise(resolve => setTimeout(resolve, 10)); + return ++counter; + }; + + const result1 = await deduplicator.run(task); + await new Promise(resolve => setTimeout(resolve, 20)); + const result2 = await deduplicator.run(task); + + expect(result1).toBe(1); + expect(result2).toBe(2); + }); + }); +}); diff --git a/packages/foam-vscode/src/core/utils/task-deduplicator.ts b/packages/foam-vscode/src/core/utils/task-deduplicator.ts new file mode 100644 index 00000000..5abe8920 --- /dev/null +++ b/packages/foam-vscode/src/core/utils/task-deduplicator.ts @@ -0,0 +1,67 @@ +/** + * A utility class for deduplicating concurrent async operations. + * When multiple calls are made while a task is running, subsequent calls + * will wait for and receive the result of the already-running task instead + * of starting a new one. + * + * @example + * const deduplicator = new TaskDeduplicator(); + * + * async function expensiveOperation(input: string): Promise { + * return deduplicator.run(async () => { + * // Expensive work here + * return result; + * }); + * } + * + * // Multiple concurrent calls will share the same execution + * const [result1, result2] = await Promise.all([ + * expensiveOperation("test"), + * expensiveOperation("test"), + * ]); + * // Only runs once, both get the same result + */ +export class TaskDeduplicator { + private runningTask: Promise | null = null; + + /** + * Run a task with deduplication. + * If a task is already running, waits for it to complete and returns its result. + * Otherwise, starts the task and stores its promise for other callers to await. + * + * @param task The async function to execute + * @param onDuplicate Optional callback when a duplicate call is detected + * @returns The result of the task + */ + async run(task: () => Promise, onDuplicate?: () => void): Promise { + // If already running, wait for the existing task + if (this.runningTask) { + onDuplicate?.(); + return await this.runningTask; + } + + // Start the task and store the promise + this.runningTask = task(); + + try { + return await this.runningTask; + } finally { + // Clear the task when done + this.runningTask = null; + } + } + + /** + * Check if a task is currently running + */ + isRunning(): boolean { + return this.runningTask !== null; + } + + /** + * Clear the running task reference (useful for testing or error recovery) + */ + clear(): void { + this.runningTask = null; + } +} diff --git a/packages/foam-vscode/src/extension.ts b/packages/foam-vscode/src/extension.ts index 64e3d8d8..642f4272 100644 --- a/packages/foam-vscode/src/extension.ts +++ b/packages/foam-vscode/src/extension.ts @@ -19,6 +19,7 @@ import { VsCodeWatcher } from './services/watcher'; import { createMarkdownParser } from './core/services/markdown-parser'; import VsCodeBasedParserCache from './services/cache'; import { createMatcherAndDataStore } from './services/editor'; +import { OllamaEmbeddingProvider } from './ai/providers/ollama/ollama-provider'; export async function activate(context: ExtensionContext) { const logger = new VsCodeOutputLogger(); @@ -71,13 +72,20 @@ export async function activate(context: ExtensionContext) { attachmentExtConfig ); + // Initialize embedding provider + const aiEnabled = workspace.getConfiguration('foam.experimental').get('ai'); + const embeddingProvider = aiEnabled + ? new OllamaEmbeddingProvider() + : undefined; + const foamPromise = bootstrap( matcher, watcher, dataStore, parser, [markdownProvider, attachmentProvider], - defaultExtension + defaultExtension, + embeddingProvider ); // Load the features diff --git a/packages/foam-vscode/src/features/commands/index.ts b/packages/foam-vscode/src/features/commands/index.ts index 4c916338..85fdbb56 100644 --- a/packages/foam-vscode/src/features/commands/index.ts +++ b/packages/foam-vscode/src/features/commands/index.ts @@ -13,3 +13,5 @@ export { default as createNote } from './create-note'; export { default as searchTagCommand } from './search-tag'; export { default as renameTagCommand } from './rename-tag'; export { default as convertLinksCommand } from './convert-links'; +export { default as showSimilarNotesCommand } from '../../ai/vscode/commands/show-similar-notes'; +export { default as buildEmbeddingsCommand } from '../../ai/vscode/commands/build-embeddings'; diff --git a/packages/foam-vscode/src/features/panels/index.ts b/packages/foam-vscode/src/features/panels/index.ts index 57460546..2ecced0f 100644 --- a/packages/foam-vscode/src/features/panels/index.ts +++ b/packages/foam-vscode/src/features/panels/index.ts @@ -4,3 +4,4 @@ export { default as orphans } from './orphans'; export { default as placeholders } from './placeholders'; export { default as tags } from './tags-explorer'; export { default as notes } from './notes-explorer'; +export { default as relatedNotes } from '../../ai/vscode/panels/related-notes'; diff --git a/packages/foam-vscode/src/test/test-utils-vscode.ts b/packages/foam-vscode/src/test/test-utils-vscode.ts index 74e082f9..512f0212 100644 --- a/packages/foam-vscode/src/test/test-utils-vscode.ts +++ b/packages/foam-vscode/src/test/test-utils-vscode.ts @@ -80,7 +80,9 @@ export const waitForNoteInFoamWorkspace = async (uri: URI, timeout = 5000) => { } await wait(100); } - return false; + throw new Error( + `Timeout waiting for note ${uri.toString()} in Foam workspace` + ); }; /** diff --git a/packages/foam-vscode/src/test/test-utils.ts b/packages/foam-vscode/src/test/test-utils.ts index fb01c028..cf83a6d3 100644 --- a/packages/foam-vscode/src/test/test-utils.ts +++ b/packages/foam-vscode/src/test/test-utils.ts @@ -9,11 +9,49 @@ import { FoamWorkspace } from '../core/model/workspace'; import { MarkdownResourceProvider } from '../core/services/markdown-provider'; import { Resource } from '../core/model/note'; import { createMarkdownParser } from '../core/services/markdown-parser'; +import { IDataStore } from '../core/services/datastore'; export { default as waitForExpect } from 'wait-for-expect'; Logger.setLevel('error'); +/** + * An in-memory data store for testing that stores file content in a Map. + * This allows tests to provide text content for notes without touching the filesystem. + */ +export class InMemoryDataStore implements IDataStore { + private files = new Map(); + + /** + * Set the content for a file + */ + set(uri: URI, content: string): void { + this.files.set(uri.path, content); + } + + /** + * Delete a file + */ + delete(uri: URI): void { + this.files.delete(uri.path); + } + + /** + * Clear all files + */ + clear(): void { + this.files.clear(); + } + + async list(): Promise { + return Array.from(this.files.keys()).map(path => URI.parse(path, 'file')); + } + + async read(uri: URI): Promise { + return this.files.get(uri.path) ?? null; + } +} + export const TEST_DATA_DIR = URI.file(__dirname).joinPath( '..', '..', @@ -29,11 +67,14 @@ const position = Range.create(0, 0, 0, 100); */ export const strToUri = URI.file; -export const createTestWorkspace = (workspaceRoots: URI[] = []) => { +export const createTestWorkspace = ( + workspaceRoots: URI[] = [], + dataStore?: IDataStore +) => { const workspace = new FoamWorkspace(); const parser = createMarkdownParser(); const provider = new MarkdownResourceProvider( - { + dataStore ?? { read: _ => Promise.resolve(''), list: () => Promise.resolve([]), }, @@ -51,7 +92,6 @@ export const createTestNote = (params: { links?: Array<{ slug: string; definitionUrl?: string } | { to: string }>; tags?: string[]; aliases?: string[]; - text?: string; sections?: string[]; root?: URI; type?: string; diff --git a/packages/foam-vscode/src/test/vscode-mock.ts b/packages/foam-vscode/src/test/vscode-mock.ts index 3431d134..85b20fe2 100644 --- a/packages/foam-vscode/src/test/vscode-mock.ts +++ b/packages/foam-vscode/src/test/vscode-mock.ts @@ -17,10 +17,12 @@ import { createMarkdownParser } from '../core/services/markdown-parser'; import { GenericDataStore, AlwaysIncludeMatcher, + IWatcher, } from '../core/services/datastore'; import { MarkdownResourceProvider } from '../core/services/markdown-provider'; import { randomString } from './test-utils'; import micromatch from 'micromatch'; +import { Emitter } from '../core/common/event'; interface Thenable { then( @@ -247,6 +249,13 @@ export function createVSCodeUri(foamUri: URI): Uri { }; } +/** + * Convert VS Code Uri to Foam URI + */ +export function fromVsCodeUri(vsCodeUri: Uri): URI { + return URI.file(vsCodeUri.fsPath); +} + // VS Code Uri static methods // eslint-disable-next-line @typescript-eslint/no-redeclare export const Uri = { @@ -423,6 +432,12 @@ export enum DiagnosticSeverity { Hint = 3, } +export enum ProgressLocation { + SourceControl = 1, + Window = 10, + Notification = 15, +} + // ===== Code Actions ===== export class CodeActionKind { @@ -588,6 +603,57 @@ export interface Disposable { dispose(): void; } +// ===== Cancellation ===== + +export interface CancellationToken { + readonly isCancellationRequested: boolean; + readonly onCancellationRequested: Event; +} + +export class CancellationTokenSource { + private _token: CancellationToken | undefined; + private _emitter: EventEmitter | undefined; + private _isCancelled = false; + + get token(): CancellationToken { + if (!this._token) { + this._emitter = new EventEmitter(); + this._token = { + isCancellationRequested: this._isCancelled, + onCancellationRequested: this._emitter.event, + }; + } + return this._token; + } + + cancel(): void { + if (!this._isCancelled) { + this._isCancelled = true; + if (this._emitter) { + this._emitter.fire(undefined); + } + // Update token state + if (this._token) { + (this._token as any).isCancellationRequested = true; + } + } + } + + dispose(): void { + if (this._emitter) { + this._emitter.dispose(); + this._emitter = undefined; + } + this._token = undefined; + } +} + +// ===== Progress ===== + +export interface Progress { + report(value: T): void; +} + export class EventEmitter { private listeners: ((e: T) => any)[] = []; @@ -791,11 +857,21 @@ class MockTextDocument implements TextDocument { this._content = content; // Write the content to file if provided try { + const existed = fs.existsSync(uri.fsPath); const dir = path.dirname(uri.fsPath); if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } fs.writeFileSync(uri.fsPath, content); + + // Manually fire watcher events (can't use async workspace.fs in constructor) + for (const watcher of mockState.fileWatchers) { + if (existed) { + watcher._fireChange(uri); + } else { + watcher._fireCreate(uri); + } + } } catch (error) { // Ignore write errors in mock } @@ -1115,10 +1191,31 @@ class MockFileSystem implements FileSystem { } async writeFile(uri: Uri, content: Uint8Array): Promise { + // Check if file exists before writing + const existed = await this.exists(uri); + // Ensure directory exists const dir = path.dirname(uri.fsPath); await fs.promises.mkdir(dir, { recursive: true }); await fs.promises.writeFile(uri.fsPath, content); + + // Fire watcher events + for (const watcher of mockState.fileWatchers) { + if (existed) { + watcher._fireChange(uri); + } else { + watcher._fireCreate(uri); + } + } + } + + private async exists(uri: Uri): Promise { + try { + await fs.promises.access(uri.fsPath); + return true; + } catch { + return false; + } } async delete(uri: Uri, options?: { recursive?: boolean }): Promise { @@ -1133,6 +1230,11 @@ class MockFileSystem implements FileSystem { } else { await fs.promises.unlink(uri.fsPath); } + + // Fire watcher events + for (const watcher of mockState.fileWatchers) { + watcher._fireDelete(uri); + } } async stat( @@ -1175,6 +1277,84 @@ class MockFileSystem implements FileSystem { options?: { overwrite?: boolean } ): Promise { await fs.promises.rename(source.fsPath, target.fsPath); + + // Fire watcher events (rename = delete + create) + for (const watcher of mockState.fileWatchers) { + watcher._fireDelete(source); + watcher._fireCreate(target); + } + } +} + +// ===== File System Watcher ===== + +export interface FileSystemWatcher extends Disposable { + onDidCreate: Event; + onDidChange: Event; + onDidDelete: Event; + ignoreCreateEvents: boolean; + ignoreChangeEvents: boolean; + ignoreDeleteEvents: boolean; +} + +class MockFileSystemWatcher implements FileSystemWatcher { + private onDidCreateEmitter = new Emitter(); + private onDidChangeEmitter = new Emitter(); + private onDidDeleteEmitter = new Emitter(); + + onDidCreate = this.onDidCreateEmitter.event; + onDidChange = this.onDidChangeEmitter.event; + onDidDelete = this.onDidDeleteEmitter.event; + + ignoreCreateEvents = false; + ignoreChangeEvents = false; + ignoreDeleteEvents = false; + + constructor(private pattern: string) { + // Register this watcher in mockState (will be added to mockState) + if (mockState.fileWatchers) { + mockState.fileWatchers.push(this); + } + } + + // Internal methods called by MockFileSystem + _fireCreate(uri: Uri) { + if (!this.ignoreCreateEvents && this.matches(uri)) { + this.onDidCreateEmitter.fire(uri); + } + } + + _fireChange(uri: Uri) { + if (!this.ignoreChangeEvents && this.matches(uri)) { + this.onDidChangeEmitter.fire(uri); + } + } + + _fireDelete(uri: Uri) { + if (!this.ignoreDeleteEvents && this.matches(uri)) { + this.onDidDeleteEmitter.fire(uri); + } + } + + private matches(uri: Uri): boolean { + const workspaceFolder = mockState.workspaceFolders[0]; + if (!workspaceFolder) return false; + + const relativePath = path.relative(workspaceFolder.uri.fsPath, uri.fsPath); + // Use micromatch (already imported) for glob matching + return micromatch.isMatch(relativePath, this.pattern); + } + + dispose() { + if (mockState.fileWatchers) { + const index = mockState.fileWatchers.indexOf(this); + if (index >= 0) { + mockState.fileWatchers.splice(index, 1); + } + } + this.onDidCreateEmitter.dispose(); + this.onDidChangeEmitter.dispose(); + this.onDidDeleteEmitter.dispose(); } } @@ -1343,17 +1523,41 @@ class TestFoam { // Create resource providers const providers = [new MarkdownResourceProvider(dataStore, parser)]; - // Use the bootstrap function without file watcher (simpler for tests) + // Create file watcher for automatic workspace updates + const vsCodeWatcher = workspace.createFileSystemWatcher('**/*'); + + // Convert VS Code Uri events to Foam URI events + const onDidCreateEmitter = new Emitter(); + const onDidChangeEmitter = new Emitter(); + const onDidDeleteEmitter = new Emitter(); + + vsCodeWatcher.onDidCreate(uri => + onDidCreateEmitter.fire(fromVsCodeUri(uri)) + ); + vsCodeWatcher.onDidChange(uri => + onDidChangeEmitter.fire(fromVsCodeUri(uri)) + ); + vsCodeWatcher.onDidDelete(uri => + onDidDeleteEmitter.fire(fromVsCodeUri(uri)) + ); + + const foamWatcher: IWatcher = { + onDidCreate: onDidCreateEmitter.event, + onDidChange: onDidChangeEmitter.event, + onDidDelete: onDidDeleteEmitter.event, + }; + + // Use the bootstrap function with file watcher const foam = await bootstrap( matcher, - undefined, + foamWatcher, dataStore, parser, providers, '.md' ); - Logger.info('Mock Foam instance created (manual reload for tests)'); + Logger.info('Mock Foam instance created with file watcher'); return foam; } @@ -1399,13 +1603,14 @@ async function initializeFoamCommands(foam: Foam): Promise { await foamCommands.updateWikilinksCommand(mockContext, foamPromise); await foamCommands.openDailyNoteForDateCommand(mockContext, foamPromise); await foamCommands.convertLinksCommand(mockContext, foamPromise); + await foamCommands.buildEmbeddingsCommand(mockContext, foamPromise); + await foamCommands.openDailyNoteCommand(mockContext, foamPromise); + await foamCommands.openDatedNote(mockContext, foamPromise); // Commands that only need context await foamCommands.copyWithoutBracketsCommand(mockContext); await foamCommands.createFromTemplateCommand(mockContext); await foamCommands.createNewTemplate(mockContext); - await foamCommands.openDailyNoteCommand(mockContext, foamPromise); - await foamCommands.openDatedNote(mockContext, foamPromise); Logger.info('Foam commands initialized successfully in mock environment'); } @@ -1420,6 +1625,7 @@ const mockState = { commands: new Map any>(), fileSystem: new MockFileSystem(), configuration: new MockWorkspaceConfiguration(), + fileWatchers: [] as MockFileSystemWatcher[], }; // Window namespace @@ -1514,6 +1720,31 @@ export const window = { message ); }, + + async withProgress( + options: { + location: ProgressLocation; + title?: string; + cancellable?: boolean; + }, + task: ( + progress: Progress<{ message?: string; increment?: number }>, + token: CancellationToken + ) => Thenable + ): Promise { + const tokenSource = new CancellationTokenSource(); + const progress: Progress<{ message?: string; increment?: number }> = { + report: () => { + // No-op in mock, but can be overridden in tests + }, + }; + + try { + return await task(progress, tokenSource.token); + } finally { + tokenSource.dispose(); + } + }, }; // Workspace namespace @@ -1528,6 +1759,10 @@ export const workspace = { return mockState.fileSystem; }, + createFileSystemWatcher(globPattern: string): FileSystemWatcher { + return new MockFileSystemWatcher(globPattern); + }, + getConfiguration(section?: string): WorkspaceConfiguration { if (section) { // Return a scoped configuration for the specific section