mirror of
https://github.com/foambubble/foam.git
synced 2026-01-06 20:53:53 -05:00
add note embeddings and semantic similarity search (#1560)
* First implementation of embedding feature * Improved caching * Progress on embedding index build * Added cancellation option, and updating embeds continuously * Added "Related Notes" panel, plus various tweaks * Added task deduplicator so that notes analysis command has only one running instance * Added progress/cancel features to vscode mock * Tests * refactored code into `ai` module * Added `foam.experimental.ai` feature flag, with AI features disabled by default * `waitForNoteInFoamWorkspace` now fails on timeout * Added watcher in VS Code mock, and updated related test
This commit is contained in:
@@ -82,6 +82,13 @@
|
||||
"name": "Placeholders",
|
||||
"icon": "$(debug-disconnect)",
|
||||
"contextualTitle": "Foam"
|
||||
},
|
||||
{
|
||||
"when": "config.foam.experimental.ai",
|
||||
"id": "foam-vscode.related-notes",
|
||||
"name": "Related Notes (AI)",
|
||||
"icon": "$(sparkle)",
|
||||
"contextualTitle": "Foam"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -101,6 +108,21 @@
|
||||
{
|
||||
"view": "foam-vscode.placeholders",
|
||||
"contents": "No placeholders found for selected resource or workspace."
|
||||
},
|
||||
{
|
||||
"view": "foam-vscode.related-notes",
|
||||
"contents": "Open a note to see related notes.",
|
||||
"when": "config.foam.experimental.ai && foam.relatedNotes.state == 'no-note'"
|
||||
},
|
||||
{
|
||||
"view": "foam-vscode.related-notes",
|
||||
"contents": "Notes haven't been analyzed yet.\n[Analyze Notes](command:foam-vscode.build-embeddings)\nAnalyze your notes to discover similar content.",
|
||||
"when": "config.foam.experimental.ai && foam.relatedNotes.state == 'no-embedding'"
|
||||
},
|
||||
{
|
||||
"view": "foam-vscode.related-notes",
|
||||
"contents": "No similar notes found for the current note.",
|
||||
"when": "config.foam.experimental.ai && foam.relatedNotes.state == 'ready'"
|
||||
}
|
||||
],
|
||||
"menus": {
|
||||
@@ -384,6 +406,16 @@
|
||||
"title": "Foam: Rename Tag",
|
||||
"icon": "$(edit)"
|
||||
},
|
||||
{
|
||||
"command": "foam-vscode.show-similar-notes",
|
||||
"title": "Foam: Show Similar Notes",
|
||||
"when": "config.foam.experimental.ai"
|
||||
},
|
||||
{
|
||||
"command": "foam-vscode.build-embeddings",
|
||||
"title": "Foam: Build Embeddings Index",
|
||||
"when": "config.foam.experimental.ai"
|
||||
},
|
||||
{
|
||||
"command": "foam-vscode.views.orphans.group-by:folder",
|
||||
"title": "Group By Folder",
|
||||
@@ -694,6 +726,7 @@
|
||||
"description": "Whether or not to navigate to the target daily note when a daily note snippet is selected."
|
||||
},
|
||||
"foam.preview.embedNoteType": {
|
||||
"when": "config.foam.experimental.ai",
|
||||
"type": "string",
|
||||
"default": "full-card",
|
||||
"enum": [
|
||||
@@ -724,7 +757,6 @@
|
||||
"default": false,
|
||||
"description": "Whether to open the graph on startup."
|
||||
}
|
||||
|
||||
}
|
||||
},
|
||||
"keybindings": [
|
||||
|
||||
17
packages/foam-vscode/src/ai/model/embedding-cache.ts
Normal file
17
packages/foam-vscode/src/ai/model/embedding-cache.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import { URI } from '../../core/model/uri';
|
||||
import { ICache } from '../../core/utils/cache';
|
||||
|
||||
type Checksum = string;
|
||||
|
||||
/**
|
||||
* Cache entry for embeddings
|
||||
*/
|
||||
export interface EmbeddingCacheEntry {
|
||||
checksum: Checksum;
|
||||
embedding: number[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache for embeddings, keyed by URI
|
||||
*/
|
||||
export type EmbeddingCache = ICache<URI, EmbeddingCacheEntry>;
|
||||
365
packages/foam-vscode/src/ai/model/embeddings.test.ts
Normal file
365
packages/foam-vscode/src/ai/model/embeddings.test.ts
Normal file
@@ -0,0 +1,365 @@
|
||||
import { FoamEmbeddings } from './embeddings';
|
||||
import {
|
||||
EmbeddingProvider,
|
||||
EmbeddingProviderInfo,
|
||||
} from '../services/embedding-provider';
|
||||
import {
|
||||
createTestWorkspace,
|
||||
InMemoryDataStore,
|
||||
waitForExpect,
|
||||
} from '../../test/test-utils';
|
||||
import { URI } from '../../core/model/uri';
|
||||
|
||||
// Helper to create a simple mock provider
|
||||
class MockProvider implements EmbeddingProvider {
|
||||
async embed(text: string): Promise<number[]> {
|
||||
const vector = new Array(384).fill(0);
|
||||
vector[0] = text.length / 100; // Deterministic based on text length
|
||||
return vector;
|
||||
}
|
||||
async isAvailable(): Promise<boolean> {
|
||||
return true;
|
||||
}
|
||||
getProviderInfo(): EmbeddingProviderInfo {
|
||||
return {
|
||||
name: 'Test Provider',
|
||||
type: 'local',
|
||||
model: { name: 'test-model', dimensions: 384 },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const ROOT = [URI.parse('/', 'file')];
|
||||
|
||||
describe('FoamEmbeddings', () => {
|
||||
describe('cosineSimilarity', () => {
|
||||
it('should return 1 for identical vectors', () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
const vector = [1, 2, 3, 4, 5];
|
||||
const similarity = embeddings.cosineSimilarity(vector, vector);
|
||||
expect(similarity).toBeCloseTo(1.0, 5);
|
||||
workspace.dispose();
|
||||
});
|
||||
|
||||
it('should return 0 for orthogonal vectors', () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
const vec1 = [1, 0, 0];
|
||||
const vec2 = [0, 1, 0];
|
||||
const similarity = embeddings.cosineSimilarity(vec1, vec2);
|
||||
expect(similarity).toBeCloseTo(0.0, 5);
|
||||
workspace.dispose();
|
||||
});
|
||||
|
||||
it('should return -1 for opposite vectors', () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
const vec1 = [1, 0, 0];
|
||||
const vec2 = [-1, 0, 0];
|
||||
const similarity = embeddings.cosineSimilarity(vec1, vec2);
|
||||
expect(similarity).toBeCloseTo(-1.0, 5);
|
||||
workspace.dispose();
|
||||
});
|
||||
|
||||
it('should return 0 for zero vectors', () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
const vec1 = [0, 0, 0];
|
||||
const vec2 = [1, 2, 3];
|
||||
const similarity = embeddings.cosineSimilarity(vec1, vec2);
|
||||
expect(similarity).toBe(0);
|
||||
workspace.dispose();
|
||||
});
|
||||
|
||||
it('should throw error for vectors of different lengths', () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
const vec1 = [1, 2, 3];
|
||||
const vec2 = [1, 2];
|
||||
expect(() => embeddings.cosineSimilarity(vec1, vec2)).toThrow();
|
||||
workspace.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
describe('updateResource', () => {
|
||||
it('should create embedding for a resource', async () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
|
||||
const noteUri = URI.parse('/path/to/note.md', 'file');
|
||||
datastore.set(noteUri, '# Test Note\n\nThis is test content');
|
||||
await workspace.fetchAndSet(noteUri);
|
||||
|
||||
await embeddings.updateResource(noteUri);
|
||||
|
||||
const embedding = embeddings.getEmbedding(noteUri);
|
||||
expect(embedding).not.toBeNull();
|
||||
expect(embedding?.length).toBe(384);
|
||||
|
||||
workspace.dispose();
|
||||
});
|
||||
|
||||
it('should remove embedding when resource is deleted', async () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
|
||||
const noteUri = URI.parse('/path/to/note.md', 'file');
|
||||
datastore.set(noteUri, '# Test Note\n\nContent');
|
||||
await workspace.fetchAndSet(noteUri);
|
||||
|
||||
await embeddings.updateResource(noteUri);
|
||||
expect(embeddings.getEmbedding(noteUri)).not.toBeNull();
|
||||
|
||||
workspace.delete(noteUri);
|
||||
await embeddings.updateResource(noteUri);
|
||||
|
||||
expect(embeddings.getEmbedding(noteUri)).toBeNull();
|
||||
|
||||
workspace.dispose();
|
||||
});
|
||||
|
||||
it('should create different embeddings for different content', async () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
|
||||
const note1Uri = URI.parse('/note1.md', 'file');
|
||||
const note2Uri = URI.parse('/note2.md', 'file');
|
||||
|
||||
// Same title, different content
|
||||
datastore.set(note1Uri, '# Same Title\n\nShort content');
|
||||
datastore.set(
|
||||
note2Uri,
|
||||
'# Same Title\n\nThis is much longer content that should produce a different embedding vector'
|
||||
);
|
||||
|
||||
await workspace.fetchAndSet(note1Uri);
|
||||
await workspace.fetchAndSet(note2Uri);
|
||||
|
||||
await embeddings.updateResource(note1Uri);
|
||||
await embeddings.updateResource(note2Uri);
|
||||
|
||||
const embedding1 = embeddings.getEmbedding(note1Uri);
|
||||
const embedding2 = embeddings.getEmbedding(note2Uri);
|
||||
|
||||
expect(embedding1).not.toBeNull();
|
||||
expect(embedding2).not.toBeNull();
|
||||
|
||||
// Embeddings should be different because content is different
|
||||
// Our mock provider uses text.length for the first vector component
|
||||
expect(embedding1![0]).not.toBe(embedding2![0]);
|
||||
|
||||
workspace.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
describe('hasEmbeddings', () => {
|
||||
it('should return false when no embeddings exist', () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
expect(embeddings.hasEmbeddings()).toBe(false);
|
||||
workspace.dispose();
|
||||
});
|
||||
|
||||
it('should return true when embeddings exist', async () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
|
||||
const noteUri = URI.parse('/path/to/note.md', 'file');
|
||||
datastore.set(noteUri, '# Note\n\nContent');
|
||||
await workspace.fetchAndSet(noteUri);
|
||||
|
||||
await embeddings.updateResource(noteUri);
|
||||
|
||||
expect(embeddings.hasEmbeddings()).toBe(true);
|
||||
|
||||
workspace.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getSimilar', () => {
|
||||
it('should return empty array when no embedding exists for target', () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
const uri = URI.parse('/path/to/note.md', 'file');
|
||||
|
||||
const similar = embeddings.getSimilar(uri, 5);
|
||||
|
||||
expect(similar).toEqual([]);
|
||||
workspace.dispose();
|
||||
});
|
||||
|
||||
it('should return similar notes sorted by similarity', async () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
|
||||
// Create notes with different content lengths
|
||||
const note1Uri = URI.parse('/note1.md', 'file');
|
||||
const note2Uri = URI.parse('/note2.md', 'file');
|
||||
const note3Uri = URI.parse('/note3.md', 'file');
|
||||
|
||||
datastore.set(note1Uri, '# Note 1\n\nShort');
|
||||
datastore.set(note2Uri, '# Note 2\n\nMedium length text');
|
||||
datastore.set(note3Uri, '# Note 3\n\nVery long text content here');
|
||||
|
||||
await workspace.fetchAndSet(note1Uri);
|
||||
await workspace.fetchAndSet(note2Uri);
|
||||
await workspace.fetchAndSet(note3Uri);
|
||||
|
||||
await embeddings.updateResource(note1Uri);
|
||||
await embeddings.updateResource(note2Uri);
|
||||
await embeddings.updateResource(note3Uri);
|
||||
|
||||
// Get similar to note2
|
||||
const similar = embeddings.getSimilar(note2Uri, 10);
|
||||
|
||||
expect(similar.length).toBe(2); // Excludes self
|
||||
expect(similar[0].uri.path).toBeTruthy();
|
||||
expect(similar[0].similarity).toBeGreaterThanOrEqual(
|
||||
similar[1].similarity
|
||||
);
|
||||
|
||||
workspace.dispose();
|
||||
});
|
||||
|
||||
it('should respect topK parameter', async () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
|
||||
// Create multiple notes
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const noteUri = URI.parse(`/note${i}.md`, 'file');
|
||||
datastore.set(noteUri, `# Note ${i}\n\nContent ${i}`);
|
||||
await workspace.fetchAndSet(noteUri);
|
||||
await embeddings.updateResource(noteUri);
|
||||
}
|
||||
|
||||
const target = URI.parse('/note0.md', 'file');
|
||||
const similar = embeddings.getSimilar(target, 3);
|
||||
|
||||
expect(similar.length).toBe(3);
|
||||
|
||||
workspace.dispose();
|
||||
});
|
||||
|
||||
it('should not include self in similar results', async () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = new FoamEmbeddings(workspace, new MockProvider());
|
||||
|
||||
const noteUri = URI.parse('/note.md', 'file');
|
||||
datastore.set(noteUri, '# Note\n\nContent');
|
||||
await workspace.fetchAndSet(noteUri);
|
||||
await embeddings.updateResource(noteUri);
|
||||
|
||||
const similar = embeddings.getSimilar(noteUri, 10);
|
||||
|
||||
expect(similar.find(s => s.uri.path === noteUri.path)).toBeUndefined();
|
||||
|
||||
workspace.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
describe('fromWorkspace with monitoring', () => {
|
||||
it('should automatically update when resource is added', async () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const embeddings = FoamEmbeddings.fromWorkspace(
|
||||
workspace,
|
||||
new MockProvider(),
|
||||
true
|
||||
);
|
||||
|
||||
const noteUri = URI.parse('/new-note.md', 'file');
|
||||
datastore.set(noteUri, '# New Note\n\nContent');
|
||||
await workspace.fetchAndSet(noteUri);
|
||||
|
||||
// Give it a moment to process
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const embedding = embeddings.getEmbedding(noteUri);
|
||||
expect(embedding).not.toBeNull();
|
||||
|
||||
embeddings.dispose();
|
||||
workspace.dispose();
|
||||
});
|
||||
|
||||
it('should automatically update when resource is modified', async () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const noteUri = URI.parse('/note.md', 'file');
|
||||
|
||||
datastore.set(noteUri, '# Note\n\nOriginal content');
|
||||
await workspace.fetchAndSet(noteUri);
|
||||
|
||||
const embeddings = FoamEmbeddings.fromWorkspace(
|
||||
workspace,
|
||||
new MockProvider(),
|
||||
true
|
||||
);
|
||||
|
||||
await embeddings.updateResource(noteUri);
|
||||
const originalEmbedding = embeddings.getEmbedding(noteUri);
|
||||
|
||||
// Update the content of the note to simulate a change
|
||||
datastore.set(noteUri, '# Note\n\nDifferent content that is much longer');
|
||||
|
||||
// Trigger workspace update event
|
||||
await workspace.fetchAndSet(noteUri);
|
||||
|
||||
// Wait for automatic update
|
||||
await waitForExpect(
|
||||
() => {
|
||||
const newEmbedding = embeddings.getEmbedding(noteUri);
|
||||
expect(newEmbedding).not.toEqual(originalEmbedding);
|
||||
},
|
||||
1000,
|
||||
50
|
||||
);
|
||||
|
||||
embeddings.dispose();
|
||||
workspace.dispose();
|
||||
});
|
||||
|
||||
it('should automatically remove embedding when resource is deleted', async () => {
|
||||
const datastore = new InMemoryDataStore();
|
||||
const workspace = createTestWorkspace(ROOT, datastore);
|
||||
const noteUri = URI.parse('/note.md', 'file');
|
||||
|
||||
datastore.set(noteUri, '# Note\n\nContent');
|
||||
await workspace.fetchAndSet(noteUri);
|
||||
|
||||
const embeddings = FoamEmbeddings.fromWorkspace(
|
||||
workspace,
|
||||
new MockProvider(),
|
||||
true
|
||||
);
|
||||
|
||||
await embeddings.updateResource(noteUri);
|
||||
expect(embeddings.getEmbedding(noteUri)).not.toBeNull();
|
||||
|
||||
workspace.delete(noteUri);
|
||||
|
||||
// Give it a moment to process
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
|
||||
expect(embeddings.getEmbedding(noteUri)).toBeNull();
|
||||
|
||||
embeddings.dispose();
|
||||
workspace.dispose();
|
||||
});
|
||||
});
|
||||
});
|
||||
382
packages/foam-vscode/src/ai/model/embeddings.ts
Normal file
382
packages/foam-vscode/src/ai/model/embeddings.ts
Normal file
@@ -0,0 +1,382 @@
|
||||
import { Emitter } from '../../core/common/event';
|
||||
import { IDisposable } from '../../core/common/lifecycle';
|
||||
import { Logger } from '../../core/utils/log';
|
||||
import { hash } from '../../core/utils';
|
||||
import { EmbeddingProvider, Embedding } from '../services/embedding-provider';
|
||||
import { EmbeddingCache } from './embedding-cache';
|
||||
import {
|
||||
ProgressCallback,
|
||||
CancellationToken,
|
||||
CancellationError,
|
||||
} from '../../core/services/progress';
|
||||
import { FoamWorkspace } from '../../core/model/workspace';
|
||||
import { URI } from '../../core/model/uri';
|
||||
|
||||
/**
|
||||
* Represents a similar resource with its similarity score
|
||||
*/
|
||||
export interface SimilarResource {
|
||||
uri: URI;
|
||||
similarity: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Context information for embedding progress
|
||||
*/
|
||||
export interface EmbeddingProgressContext {
|
||||
/** URI of the current resource */
|
||||
uri: URI;
|
||||
/** Title of the current resource */
|
||||
title: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Manages embeddings for all resources in the workspace
|
||||
*/
|
||||
export class FoamEmbeddings implements IDisposable {
|
||||
/**
|
||||
* Maps resource URIs to their embeddings
|
||||
*/
|
||||
private embeddings: Map<string, Embedding> = new Map();
|
||||
|
||||
private onDidUpdateEmitter = new Emitter<void>();
|
||||
onDidUpdate = this.onDidUpdateEmitter.event;
|
||||
|
||||
/**
|
||||
* List of disposables to destroy with the embeddings
|
||||
*/
|
||||
private disposables: IDisposable[] = [];
|
||||
|
||||
constructor(
|
||||
private readonly workspace: FoamWorkspace,
|
||||
private readonly provider: EmbeddingProvider,
|
||||
private readonly cache?: EmbeddingCache
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Get the embedding for a resource
|
||||
* @param uri The URI of the resource
|
||||
* @returns The embedding vector, or null if not found
|
||||
*/
|
||||
public getEmbedding(uri: URI): number[] | null {
|
||||
const embedding = this.embeddings.get(uri.path);
|
||||
return embedding ? embedding.vector : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if embeddings are available
|
||||
* @returns true if at least one embedding exists
|
||||
*/
|
||||
public hasEmbeddings(): boolean {
|
||||
return this.embeddings.size > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of embeddings
|
||||
* @returns The count of embeddings
|
||||
*/
|
||||
public size(): number {
|
||||
return this.embeddings.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find similar resources to a given resource
|
||||
* @param uri The URI of the target resource
|
||||
* @param topK The number of similar resources to return
|
||||
* @returns Array of similar resources sorted by similarity (highest first)
|
||||
*/
|
||||
public getSimilar(uri: URI, topK: number = 10): SimilarResource[] {
|
||||
const targetEmbedding = this.getEmbedding(uri);
|
||||
if (!targetEmbedding) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const similarities: SimilarResource[] = [];
|
||||
|
||||
for (const [path, embedding] of this.embeddings.entries()) {
|
||||
// Skip self
|
||||
if (path === uri.path) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const similarity = this.cosineSimilarity(
|
||||
targetEmbedding,
|
||||
embedding.vector
|
||||
);
|
||||
similarities.push({
|
||||
uri: URI.file(path),
|
||||
similarity,
|
||||
});
|
||||
}
|
||||
|
||||
// Sort by similarity (highest first) and take top K
|
||||
similarities.sort((a, b) => b.similarity - a.similarity);
|
||||
return similarities.slice(0, topK);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate cosine similarity between two vectors
|
||||
* @param a First vector
|
||||
* @param b Second vector
|
||||
* @returns Similarity score between -1 and 1 (higher is more similar)
|
||||
*/
|
||||
public cosineSimilarity(a: number[], b: number[]): number {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error('Vectors must have the same length');
|
||||
}
|
||||
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
|
||||
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
if (denominator === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return dotProduct / denominator;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update embeddings for a single resource
|
||||
* @param uri The URI of the resource to update
|
||||
* @returns The embedding vector, or null if not found/not processed
|
||||
*/
|
||||
public async updateResource(uri: URI): Promise<Embedding | null> {
|
||||
const resource = this.workspace.find(uri);
|
||||
if (!resource) {
|
||||
// Resource deleted, remove embedding
|
||||
this.embeddings.delete(uri.path);
|
||||
if (this.cache) {
|
||||
this.cache.del(uri);
|
||||
}
|
||||
this.onDidUpdateEmitter.fire();
|
||||
return null;
|
||||
}
|
||||
|
||||
// Skip non-note resources (attachments)
|
||||
if (resource.type !== 'note') {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const content = await this.workspace.readAsMarkdown(resource.uri);
|
||||
const text = this.prepareTextForEmbedding(resource.title, content);
|
||||
const textChecksum = hash(text);
|
||||
|
||||
// Check cache if available
|
||||
if (this.cache && this.cache.has(uri)) {
|
||||
const cached = this.cache.get(uri);
|
||||
if (cached.checksum === textChecksum) {
|
||||
Logger.debug(
|
||||
`Skipping embedding for ${uri.toFsPath()} - content unchanged`
|
||||
);
|
||||
// Use cached embedding
|
||||
const embedding: Embedding = {
|
||||
vector: cached.embedding,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
this.embeddings.set(uri.path, embedding);
|
||||
return embedding;
|
||||
}
|
||||
}
|
||||
|
||||
// Generate new embedding
|
||||
const vector = await this.provider.embed(text);
|
||||
|
||||
const embedding: Embedding = {
|
||||
vector,
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
this.embeddings.set(uri.path, embedding);
|
||||
|
||||
// Update cache
|
||||
if (this.cache) {
|
||||
this.cache.set(uri, {
|
||||
checksum: textChecksum,
|
||||
embedding: vector,
|
||||
});
|
||||
}
|
||||
|
||||
this.onDidUpdateEmitter.fire();
|
||||
return embedding;
|
||||
} catch (error) {
|
||||
Logger.error(`Failed to update embedding for ${uri.toFsPath()}`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update embeddings for all notes, processing only missing or stale ones
|
||||
* @param onProgress Optional callback to report progress
|
||||
* @param cancellationToken Optional token to cancel the operation
|
||||
* @returns Promise that resolves when all embeddings are updated
|
||||
* @throws CancellationError if the operation is cancelled
|
||||
*/
|
||||
public async update(
|
||||
onProgress?: ProgressCallback<EmbeddingProgressContext>,
|
||||
cancellationToken?: CancellationToken
|
||||
): Promise<void> {
|
||||
const start = Date.now();
|
||||
|
||||
// Filter to only process notes (not attachments)
|
||||
const allResources = Array.from(this.workspace.resources());
|
||||
const resources = allResources.filter(r => r.type === 'note');
|
||||
|
||||
Logger.info(
|
||||
`Building embeddings for ${resources.length} notes (${allResources.length} total resources)...`
|
||||
);
|
||||
|
||||
let skipped = 0;
|
||||
let generated = 0;
|
||||
let reused = 0;
|
||||
|
||||
// Process embeddings sequentially to avoid overwhelming the service
|
||||
for (let i = 0; i < resources.length; i++) {
|
||||
// Check for cancellation
|
||||
if (cancellationToken?.isCancellationRequested) {
|
||||
Logger.info(
|
||||
`Embedding build cancelled. Processed ${i}/${resources.length} notes.`
|
||||
);
|
||||
throw new CancellationError('Embedding build cancelled');
|
||||
}
|
||||
|
||||
const resource = resources[i];
|
||||
|
||||
onProgress?.({
|
||||
current: i + 1,
|
||||
total: resources.length,
|
||||
context: {
|
||||
uri: resource.uri,
|
||||
title: resource.title,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const content = await this.workspace.readAsMarkdown(resource.uri);
|
||||
const text = this.prepareTextForEmbedding(resource.title, content);
|
||||
const textChecksum = hash(text);
|
||||
|
||||
// Check cache if available
|
||||
if (this.cache && this.cache.has(resource.uri)) {
|
||||
const cached = this.cache.get(resource.uri);
|
||||
if (cached.checksum === textChecksum) {
|
||||
// Check if we already have this embedding in memory
|
||||
const existing = this.embeddings.get(resource.uri.path);
|
||||
if (existing) {
|
||||
// Already have current embedding, skip
|
||||
reused++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Restore from cache
|
||||
this.embeddings.set(resource.uri.path, {
|
||||
vector: cached.embedding,
|
||||
createdAt: Date.now(),
|
||||
});
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Generate new embedding
|
||||
const vector = await this.provider.embed(text);
|
||||
this.embeddings.set(resource.uri.path, {
|
||||
vector,
|
||||
createdAt: Date.now(),
|
||||
});
|
||||
|
||||
// Update cache
|
||||
if (this.cache) {
|
||||
this.cache.set(resource.uri, {
|
||||
checksum: textChecksum,
|
||||
embedding: vector,
|
||||
});
|
||||
}
|
||||
|
||||
generated++;
|
||||
} catch (error) {
|
||||
Logger.error(
|
||||
`Failed to generate embedding for ${resource.uri.toFsPath()}`,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const end = Date.now();
|
||||
Logger.info(
|
||||
`Embeddings update complete: ${generated} generated, ${skipped} from cache, ${reused} already current (${
|
||||
this.embeddings.size
|
||||
}/${resources.length} total) in ${end - start}ms`
|
||||
);
|
||||
this.onDidUpdateEmitter.fire();
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare text for embedding by combining title and content
|
||||
* @param title The title of the note
|
||||
* @param content The markdown content of the note
|
||||
* @returns The combined text to embed
|
||||
*/
|
||||
private prepareTextForEmbedding(title: string, content: string): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
if (title) {
|
||||
parts.push(title);
|
||||
}
|
||||
|
||||
if (content) {
|
||||
parts.push(content);
|
||||
}
|
||||
|
||||
return parts.join('\n\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Create FoamEmbeddings from a workspace
|
||||
* @param workspace The workspace to generate embeddings for
|
||||
* @param provider The embedding provider to use
|
||||
* @param keepMonitoring Whether to automatically update embeddings when workspace changes
|
||||
* @param cache Optional cache for storing embeddings
|
||||
* @returns The FoamEmbeddings instance
|
||||
*/
|
||||
public static fromWorkspace(
|
||||
workspace: FoamWorkspace,
|
||||
provider: EmbeddingProvider,
|
||||
keepMonitoring: boolean = false,
|
||||
cache?: EmbeddingCache
|
||||
): FoamEmbeddings {
|
||||
const embeddings = new FoamEmbeddings(workspace, provider, cache);
|
||||
|
||||
if (keepMonitoring) {
|
||||
// Update embeddings when resources change
|
||||
embeddings.disposables.push(
|
||||
workspace.onDidAdd(resource => {
|
||||
embeddings.updateResource(resource.uri);
|
||||
}),
|
||||
workspace.onDidUpdate(({ new: resource }) => {
|
||||
embeddings.updateResource(resource.uri);
|
||||
}),
|
||||
workspace.onDidDelete(resource => {
|
||||
embeddings.embeddings.delete(resource.uri.path);
|
||||
embeddings.onDidUpdateEmitter.fire();
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
return embeddings;
|
||||
}
|
||||
|
||||
public dispose(): void {
|
||||
this.onDidUpdateEmitter.dispose();
|
||||
this.disposables.forEach(d => d.dispose());
|
||||
this.disposables = [];
|
||||
this.embeddings.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { URI } from '../../core/model/uri';
|
||||
import { EmbeddingCache, EmbeddingCacheEntry } from './embedding-cache';
|
||||
|
||||
/**
|
||||
* Simple in-memory implementation of embedding cache
|
||||
*/
|
||||
export class InMemoryEmbeddingCache implements EmbeddingCache {
|
||||
private cache: Map<string, EmbeddingCacheEntry> = new Map();
|
||||
|
||||
get(uri: URI): EmbeddingCacheEntry {
|
||||
return this.cache.get(uri.toString());
|
||||
}
|
||||
|
||||
has(uri: URI): boolean {
|
||||
return this.cache.has(uri.toString());
|
||||
}
|
||||
|
||||
set(uri: URI, entry: EmbeddingCacheEntry): void {
|
||||
this.cache.set(uri.toString(), entry);
|
||||
}
|
||||
|
||||
del(uri: URI): void {
|
||||
this.cache.delete(uri.toString());
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.cache.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
import { Logger } from '../../../core/utils/log';
|
||||
import {
|
||||
OllamaEmbeddingProvider,
|
||||
DEFAULT_OLLAMA_CONFIG,
|
||||
} from './ollama-provider';
|
||||
|
||||
Logger.setLevel('error');
|
||||
|
||||
describe('OllamaEmbeddingProvider', () => {
|
||||
const originalFetch = global.fetch;
|
||||
beforeEach(() => {
|
||||
global.fetch = jest.fn();
|
||||
jest.clearAllMocks();
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
global.fetch = originalFetch;
|
||||
});
|
||||
|
||||
describe('constructor', () => {
|
||||
it('should use default config when no config provided', () => {
|
||||
const provider = new OllamaEmbeddingProvider();
|
||||
const config = provider.getConfig();
|
||||
|
||||
expect(config.url).toBe(DEFAULT_OLLAMA_CONFIG.url);
|
||||
expect(config.model).toBe(DEFAULT_OLLAMA_CONFIG.model);
|
||||
expect(config.timeout).toBe(DEFAULT_OLLAMA_CONFIG.timeout);
|
||||
});
|
||||
|
||||
it('should merge custom config with defaults', () => {
|
||||
const provider = new OllamaEmbeddingProvider({
|
||||
url: 'http://custom:11434',
|
||||
});
|
||||
const config = provider.getConfig();
|
||||
|
||||
expect(config.url).toBe('http://custom:11434');
|
||||
expect(config.model).toBe(DEFAULT_OLLAMA_CONFIG.model);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getProviderInfo', () => {
|
||||
it('should return provider information', () => {
|
||||
const provider = new OllamaEmbeddingProvider();
|
||||
const info = provider.getProviderInfo();
|
||||
|
||||
expect(info.name).toBe('Ollama');
|
||||
expect(info.type).toBe('local');
|
||||
expect(info.model.name).toBe('nomic-embed-text');
|
||||
expect(info.model.dimensions).toBe(768);
|
||||
expect(info.endpoint).toBe('http://localhost:11434');
|
||||
expect(info.description).toBe('Local embedding provider using Ollama');
|
||||
expect(info.metadata).toEqual({ timeout: 30000 });
|
||||
});
|
||||
|
||||
it('should return custom model name when configured', () => {
|
||||
const provider = new OllamaEmbeddingProvider({
|
||||
model: 'custom-model',
|
||||
});
|
||||
const info = provider.getProviderInfo();
|
||||
|
||||
expect(info.model.name).toBe('custom-model');
|
||||
});
|
||||
|
||||
it('should return custom endpoint when configured', () => {
|
||||
const provider = new OllamaEmbeddingProvider({
|
||||
url: 'http://custom:8080',
|
||||
});
|
||||
const info = provider.getProviderInfo();
|
||||
|
||||
expect(info.endpoint).toBe('http://custom:8080');
|
||||
});
|
||||
});
|
||||
|
||||
describe('embed', () => {
|
||||
it('should successfully generate embeddings', async () => {
|
||||
const mockEmbedding = new Array(768).fill(0.1);
|
||||
(global.fetch as jest.Mock).mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({ embeddings: [mockEmbedding] }),
|
||||
});
|
||||
|
||||
const provider = new OllamaEmbeddingProvider();
|
||||
const result = await provider.embed('test text');
|
||||
|
||||
expect(result).toEqual(mockEmbedding);
|
||||
expect(global.fetch).toHaveBeenCalledWith(
|
||||
'http://localhost:11434/api/embed',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: 'nomic-embed-text',
|
||||
input: ['test text'],
|
||||
}),
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw error on non-ok response', async () => {
|
||||
(global.fetch as jest.Mock).mockResolvedValueOnce({
|
||||
ok: false,
|
||||
status: 500,
|
||||
text: async () => 'Internal server error',
|
||||
});
|
||||
|
||||
const provider = new OllamaEmbeddingProvider();
|
||||
|
||||
await expect(provider.embed('test')).rejects.toThrow(
|
||||
'AI service error (500)'
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw error on connection refused', async () => {
|
||||
(global.fetch as jest.Mock).mockRejectedValueOnce(
|
||||
new Error('fetch failed')
|
||||
);
|
||||
|
||||
const provider = new OllamaEmbeddingProvider();
|
||||
|
||||
await expect(provider.embed('test')).rejects.toThrow(
|
||||
'Cannot connect to Ollama'
|
||||
);
|
||||
});
|
||||
|
||||
it('should timeout after configured duration', async () => {
|
||||
(global.fetch as jest.Mock).mockImplementationOnce(
|
||||
(_url, options) =>
|
||||
new Promise((_resolve, reject) => {
|
||||
// Simulate abort signal being triggered
|
||||
options.signal.addEventListener('abort', () => {
|
||||
const error = new Error('The operation was aborted');
|
||||
error.name = 'AbortError';
|
||||
reject(error);
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
const provider = new OllamaEmbeddingProvider({ timeout: 1000 });
|
||||
const embedPromise = provider.embed('test');
|
||||
|
||||
// Fast-forward time to trigger timeout
|
||||
jest.advanceTimersByTime(1001);
|
||||
|
||||
await expect(embedPromise).rejects.toThrow('AI service took too long');
|
||||
});
|
||||
});
|
||||
|
||||
describe('isAvailable', () => {
|
||||
it('should return true when Ollama is available', async () => {
|
||||
(global.fetch as jest.Mock).mockResolvedValueOnce({
|
||||
ok: true,
|
||||
});
|
||||
|
||||
const provider = new OllamaEmbeddingProvider();
|
||||
const result = await provider.isAvailable();
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(global.fetch).toHaveBeenCalledWith(
|
||||
'http://localhost:11434/api/tags',
|
||||
expect.objectContaining({
|
||||
method: 'GET',
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should return false when Ollama is not available', async () => {
|
||||
(global.fetch as jest.Mock).mockRejectedValueOnce(
|
||||
new Error('Connection refused')
|
||||
);
|
||||
|
||||
const provider = new OllamaEmbeddingProvider();
|
||||
const result = await provider.isAvailable();
|
||||
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it('should return false when Ollama returns non-ok status', async () => {
|
||||
(global.fetch as jest.Mock).mockResolvedValueOnce({
|
||||
ok: false,
|
||||
status: 404,
|
||||
});
|
||||
|
||||
const provider = new OllamaEmbeddingProvider();
|
||||
const result = await provider.isAvailable();
|
||||
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it('should timeout quickly (5s) when checking availability', async () => {
|
||||
(global.fetch as jest.Mock).mockImplementationOnce(
|
||||
(_url, options) =>
|
||||
new Promise((_resolve, reject) => {
|
||||
// Simulate abort signal being triggered
|
||||
options.signal.addEventListener('abort', () => {
|
||||
const error = new Error('The operation was aborted');
|
||||
error.name = 'AbortError';
|
||||
reject(error);
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
const provider = new OllamaEmbeddingProvider();
|
||||
const availabilityPromise = provider.isAvailable();
|
||||
|
||||
// Fast-forward time to trigger timeout (5s for availability check)
|
||||
jest.advanceTimersByTime(5001);
|
||||
|
||||
const result = await availabilityPromise;
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('OllamaEmbeddingProvider - Integration', () => {
|
||||
const provider = new OllamaEmbeddingProvider();
|
||||
|
||||
it('should handle text with unicode characters and emojis', async () => {
|
||||
if (!(await provider.isAvailable())) {
|
||||
console.warn('Ollama is not available, skipping test');
|
||||
return;
|
||||
}
|
||||
const text = 'Task completed ✔ 🚀: All systems go! 🌟';
|
||||
const embedding = await provider.embed(text);
|
||||
|
||||
expect(embedding).toBeDefined();
|
||||
expect(Array.isArray(embedding)).toBe(true);
|
||||
expect(embedding.length).toBe(768); // nomic-embed-text dimension
|
||||
expect(embedding.every(n => typeof n === 'number')).toBe(true);
|
||||
});
|
||||
|
||||
it('should handle text with various unicode characters', async () => {
|
||||
if (!(await provider.isAvailable())) {
|
||||
console.warn('Ollama is not available, skipping test');
|
||||
return;
|
||||
}
|
||||
const text = 'Hello 🌍 with émojis and spëcial çharacters • bullet ✓ check';
|
||||
const embedding = await provider.embed(text);
|
||||
|
||||
expect(embedding).toBeDefined();
|
||||
expect(Array.isArray(embedding)).toBe(true);
|
||||
expect(embedding.length).toBe(768);
|
||||
});
|
||||
|
||||
it('should handle text with combining unicode characters', async () => {
|
||||
if (!(await provider.isAvailable())) {
|
||||
console.warn('Ollama is not available, skipping test');
|
||||
return;
|
||||
}
|
||||
// Test with combining diacriticals that could be represented differently
|
||||
const text = 'café vs cafe\u0301'; // Two ways to represent é
|
||||
const embedding = await provider.embed(text);
|
||||
|
||||
expect(embedding).toBeDefined();
|
||||
expect(Array.isArray(embedding)).toBe(true);
|
||||
expect(embedding.length).toBe(768);
|
||||
});
|
||||
|
||||
it('should handle empty text', async () => {
|
||||
if (!(await provider.isAvailable())) {
|
||||
console.warn('Ollama is not available, skipping test');
|
||||
return;
|
||||
}
|
||||
const text = '';
|
||||
const embedding = await provider.embed(text);
|
||||
|
||||
expect(embedding).toBeDefined();
|
||||
expect(Array.isArray(embedding)).toBe(true);
|
||||
// Note: Ollama returns empty array for empty text
|
||||
expect(embedding.length).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it.each([10, 50, 60, 100, 300])(
|
||||
'should handle text of various lengths',
|
||||
async length => {
|
||||
if (!(await provider.isAvailable())) {
|
||||
console.warn('Ollama is not available, skipping test');
|
||||
return;
|
||||
}
|
||||
const text = 'Lorem ipsum dolor sit amet. '.repeat(length);
|
||||
try {
|
||||
const embedding = await provider.embed(text);
|
||||
expect(embedding).toBeDefined();
|
||||
expect(Array.isArray(embedding)).toBe(true);
|
||||
expect(embedding.length).toBe(768);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Embedding failed for text of length ${text.length}: ${error}`
|
||||
);
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
166
packages/foam-vscode/src/ai/providers/ollama/ollama-provider.ts
Normal file
166
packages/foam-vscode/src/ai/providers/ollama/ollama-provider.ts
Normal file
@@ -0,0 +1,166 @@
|
||||
import {
|
||||
EmbeddingProvider,
|
||||
EmbeddingProviderInfo,
|
||||
} from '../../services/embedding-provider';
|
||||
import { Logger } from '../../../core/utils/log';
|
||||
|
||||
/**
|
||||
* Configuration for Ollama embedding provider
|
||||
*/
|
||||
export interface OllamaConfig {
|
||||
/** Base URL for Ollama API (default: http://localhost:11434) */
|
||||
url: string;
|
||||
/** Model name to use for embeddings (default: nomic-embed-text) */
|
||||
model: string;
|
||||
/** Request timeout in milliseconds (default: 30000) */
|
||||
timeout: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default configuration for Ollama
|
||||
*/
|
||||
export const DEFAULT_OLLAMA_CONFIG: OllamaConfig = {
|
||||
url: 'http://localhost:11434',
|
||||
model: 'nomic-embed-text',
|
||||
timeout: 30000,
|
||||
};
|
||||
|
||||
/**
|
||||
* Ollama API response for embeddings
|
||||
*/
|
||||
interface OllamaEmbeddingResponse {
|
||||
embeddings: number[][];
|
||||
}
|
||||
|
||||
/**
|
||||
* Embedding provider that uses Ollama for generating embeddings
|
||||
*/
|
||||
export class OllamaEmbeddingProvider implements EmbeddingProvider {
|
||||
private config: OllamaConfig;
|
||||
|
||||
constructor(config: Partial<OllamaConfig> = {}) {
|
||||
this.config = { ...DEFAULT_OLLAMA_CONFIG, ...config };
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate an embedding for the given text
|
||||
*/
|
||||
async embed(text: string): Promise<number[]> {
|
||||
// normalize text to suitable input (format and size)
|
||||
// TODO we should better handle long texts by chunking them and averaging embeddings
|
||||
const input = text.substring(0, 6000).normalize();
|
||||
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(
|
||||
() => controller.abort(),
|
||||
this.config.timeout
|
||||
);
|
||||
|
||||
const response = await fetch(`${this.config.url}/api/embed`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.config.model,
|
||||
input: [input],
|
||||
}),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`AI service error (${response.status}): ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
if (data.embeddings == null) {
|
||||
throw new Error(
|
||||
`Invalid response from AI service: ${JSON.stringify(data)}`
|
||||
);
|
||||
}
|
||||
return data.embeddings[0];
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
if (error.name === 'AbortError') {
|
||||
throw new Error(
|
||||
'AI service took too long to respond. It may be busy processing another request.'
|
||||
);
|
||||
}
|
||||
if (
|
||||
error.message.includes('fetch') ||
|
||||
error.message.includes('ECONNREFUSED')
|
||||
) {
|
||||
throw new Error(
|
||||
`Cannot connect to Ollama at ${this.config.url}. Make sure Ollama is installed and running.`
|
||||
);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if Ollama is available and the model is accessible
|
||||
*/
|
||||
async isAvailable(): Promise<boolean> {
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
||||
|
||||
// Try to reach the Ollama API
|
||||
const response = await fetch(`${this.config.url}/api/tags`, {
|
||||
method: 'GET',
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!response.ok) {
|
||||
Logger.warn(
|
||||
`Ollama API returned status ${response.status} when checking availability`
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
Logger.debug(
|
||||
`Ollama not available at ${this.config.url}: ${
|
||||
error instanceof Error ? error.message : 'Unknown error'
|
||||
}`
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get provider information including model details
|
||||
*/
|
||||
getProviderInfo(): EmbeddingProviderInfo {
|
||||
return {
|
||||
name: 'Ollama',
|
||||
type: 'local',
|
||||
model: {
|
||||
name: this.config.model,
|
||||
// nomic-embed-text produces 768-dimensional embeddings
|
||||
dimensions: 768,
|
||||
},
|
||||
description: 'Local embedding provider using Ollama',
|
||||
endpoint: this.config.url,
|
||||
metadata: {
|
||||
timeout: this.config.timeout,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current configuration
|
||||
*/
|
||||
getConfig(): OllamaConfig {
|
||||
return { ...this.config };
|
||||
}
|
||||
}
|
||||
61
packages/foam-vscode/src/ai/services/embedding-provider.ts
Normal file
61
packages/foam-vscode/src/ai/services/embedding-provider.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Information about an embedding provider and its model
|
||||
*/
|
||||
export interface EmbeddingProviderInfo {
|
||||
/** Human-readable name of the provider (e.g., "Ollama", "OpenAI") */
|
||||
name: string;
|
||||
|
||||
/** Type of provider */
|
||||
type: 'local' | 'remote';
|
||||
|
||||
/** Model information */
|
||||
model: {
|
||||
/** Model name (e.g., "nomic-embed-text", "text-embedding-3-small") */
|
||||
name: string;
|
||||
/** Vector dimensions */
|
||||
dimensions: number;
|
||||
};
|
||||
|
||||
/** Optional description of the provider */
|
||||
description?: string;
|
||||
|
||||
/** Backend endpoint/URL if applicable */
|
||||
endpoint?: string;
|
||||
|
||||
/** Additional provider-specific metadata */
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider interface for generating text embeddings
|
||||
*/
|
||||
export interface EmbeddingProvider {
|
||||
/**
|
||||
* Generate an embedding vector for the given text
|
||||
* @param text The text to embed
|
||||
* @returns A promise that resolves to the embedding vector
|
||||
*/
|
||||
embed(text: string): Promise<number[]>;
|
||||
|
||||
/**
|
||||
* Check if the embedding service is available and ready to use
|
||||
* @returns A promise that resolves to true if available, false otherwise
|
||||
*/
|
||||
isAvailable(): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* Get information about the provider and its model
|
||||
* @returns Provider metadata including name, type, model info, and configuration
|
||||
*/
|
||||
getProviderInfo(): EmbeddingProviderInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a text embedding with metadata
|
||||
*/
|
||||
export interface Embedding {
|
||||
/** The embedding vector */
|
||||
vector: number[];
|
||||
/** Timestamp when the embedding was created */
|
||||
createdAt: number;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { EmbeddingProvider, EmbeddingProviderInfo } from './embedding-provider';
|
||||
|
||||
/**
|
||||
* A no-op embedding provider that does nothing.
|
||||
* Used when no real embedding provider is available.
|
||||
*/
|
||||
export class NoOpEmbeddingProvider implements EmbeddingProvider {
|
||||
async embed(_text: string): Promise<number[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
async isAvailable(): Promise<boolean> {
|
||||
return false;
|
||||
}
|
||||
|
||||
getProviderInfo(): EmbeddingProviderInfo {
|
||||
return {
|
||||
name: 'None',
|
||||
type: 'local',
|
||||
model: {
|
||||
name: 'none',
|
||||
dimensions: 0,
|
||||
},
|
||||
description: 'No embedding provider configured',
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
/* @unit-ready */
|
||||
import * as vscode from 'vscode';
|
||||
import {
|
||||
cleanWorkspace,
|
||||
createFile,
|
||||
deleteFile,
|
||||
waitForNoteInFoamWorkspace,
|
||||
} from '../../../test/test-utils-vscode';
|
||||
import { BUILD_EMBEDDINGS_COMMAND } from './build-embeddings';
|
||||
|
||||
describe('build-embeddings command', () => {
|
||||
it('should complete successfully with no notes to analyze', async () => {
|
||||
await cleanWorkspace();
|
||||
|
||||
const showInfoSpy = jest
|
||||
.spyOn(vscode.window, 'showInformationMessage')
|
||||
.mockResolvedValue(undefined);
|
||||
|
||||
const result = await vscode.commands.executeCommand<
|
||||
'complete' | 'cancelled' | 'error'
|
||||
>(BUILD_EMBEDDINGS_COMMAND.command);
|
||||
|
||||
expect(result).toBe('complete');
|
||||
expect(showInfoSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('No notes found')
|
||||
);
|
||||
|
||||
showInfoSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('should analyze notes and report completion', async () => {
|
||||
const note1 = await createFile('# Note 1\nContent here', ['note1.md']);
|
||||
const note2 = await createFile('# Note 2\nMore content', ['note2.md']);
|
||||
|
||||
await waitForNoteInFoamWorkspace(note1.uri);
|
||||
await waitForNoteInFoamWorkspace(note2.uri);
|
||||
|
||||
const showInfoSpy = jest
|
||||
.spyOn(vscode.window, 'showInformationMessage')
|
||||
.mockResolvedValue(undefined);
|
||||
|
||||
const result = await vscode.commands.executeCommand<
|
||||
'complete' | 'cancelled' | 'error'
|
||||
>(BUILD_EMBEDDINGS_COMMAND.command);
|
||||
|
||||
expect(result).toBe('complete');
|
||||
expect(showInfoSpy).toHaveBeenCalledWith(
|
||||
expect.stringMatching(/Analyzed.*2/)
|
||||
);
|
||||
|
||||
showInfoSpy.mockRestore();
|
||||
await deleteFile(note1.uri);
|
||||
await deleteFile(note2.uri);
|
||||
});
|
||||
|
||||
it('should return cancelled status when operation is cancelled', async () => {
|
||||
const note1 = await createFile('# Note 1\nContent', ['note1.md']);
|
||||
await waitForNoteInFoamWorkspace(note1.uri);
|
||||
|
||||
const tokenSource = new vscode.CancellationTokenSource();
|
||||
|
||||
const withProgressSpy = jest
|
||||
.spyOn(vscode.window, 'withProgress')
|
||||
.mockImplementation(async (options, task) => {
|
||||
const progress = { report: () => {} };
|
||||
// Cancel immediately
|
||||
tokenSource.cancel();
|
||||
return await task(progress, tokenSource.token);
|
||||
});
|
||||
|
||||
const showInfoSpy = jest
|
||||
.spyOn(vscode.window, 'showInformationMessage')
|
||||
.mockResolvedValue(undefined);
|
||||
|
||||
const result = await vscode.commands.executeCommand<
|
||||
'complete' | 'cancelled' | 'error'
|
||||
>(BUILD_EMBEDDINGS_COMMAND.command);
|
||||
|
||||
expect(result).toBe('cancelled');
|
||||
expect(showInfoSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('cancelled')
|
||||
);
|
||||
|
||||
withProgressSpy.mockRestore();
|
||||
showInfoSpy.mockRestore();
|
||||
await deleteFile(note1.uri);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,91 @@
|
||||
import * as vscode from 'vscode';
|
||||
import { Foam } from '../../../core/model/foam';
|
||||
import { CancellationError } from '../../../core/services/progress';
|
||||
import { TaskDeduplicator } from '../../../core/utils/task-deduplicator';
|
||||
import { FoamWorkspace } from '../../../core/model/workspace';
|
||||
import { FoamEmbeddings } from '../../../ai/model/embeddings';
|
||||
|
||||
export const BUILD_EMBEDDINGS_COMMAND = {
|
||||
command: 'foam-vscode.build-embeddings',
|
||||
title: 'Foam: Analyze Notes with AI',
|
||||
};
|
||||
|
||||
export default async function activate(
|
||||
context: vscode.ExtensionContext,
|
||||
foamPromise: Promise<Foam>
|
||||
) {
|
||||
const foam = await foamPromise;
|
||||
// Deduplicate concurrent executions
|
||||
const deduplicator = new TaskDeduplicator<
|
||||
'complete' | 'cancelled' | 'error'
|
||||
>();
|
||||
|
||||
context.subscriptions.push(
|
||||
vscode.commands.registerCommand(
|
||||
BUILD_EMBEDDINGS_COMMAND.command,
|
||||
async () => {
|
||||
return await deduplicator.run(
|
||||
() => buildEmbeddings(foam.workspace, foam.embeddings),
|
||||
() => {
|
||||
vscode.window.showInformationMessage(
|
||||
'Note analysis is already in progress - waiting for it to complete'
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
async function buildEmbeddings(
|
||||
workspace: FoamWorkspace,
|
||||
embeddings: FoamEmbeddings
|
||||
): Promise<'complete' | 'cancelled' | 'error'> {
|
||||
const notesCount = workspace.list().filter(r => r.type === 'note').length;
|
||||
|
||||
if (notesCount === 0) {
|
||||
vscode.window.showInformationMessage('No notes found in workspace');
|
||||
return 'complete';
|
||||
}
|
||||
|
||||
// Show progress notification
|
||||
return await vscode.window.withProgress(
|
||||
{
|
||||
location: vscode.ProgressLocation.Window,
|
||||
title: 'Analyzing notes',
|
||||
cancellable: true,
|
||||
},
|
||||
async (progress, token) => {
|
||||
try {
|
||||
await embeddings.update(progressInfo => {
|
||||
const title = progressInfo.context?.title || 'Processing...';
|
||||
const increment = (1 / progressInfo.total) * 100;
|
||||
progress.report({
|
||||
message: `${progressInfo.current}/${progressInfo.total} - ${title}`,
|
||||
increment: increment,
|
||||
});
|
||||
}, token);
|
||||
|
||||
vscode.window.showInformationMessage(
|
||||
`✓ Analyzed ${embeddings.size()} of ${notesCount} notes`
|
||||
);
|
||||
return 'complete';
|
||||
} catch (error) {
|
||||
if (error instanceof CancellationError) {
|
||||
vscode.window.showInformationMessage(
|
||||
'Analysis cancelled. Run the command again to continue where you left off.'
|
||||
);
|
||||
return 'cancelled';
|
||||
}
|
||||
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : 'Unknown error';
|
||||
|
||||
vscode.window.showErrorMessage(
|
||||
`Failed to analyze notes: ${errorMessage}`
|
||||
);
|
||||
return 'error';
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import * as vscode from 'vscode';
|
||||
import { Foam } from '../../../core/model/foam';
|
||||
import { fromVsCodeUri, toVsCodeUri } from '../../../utils/vsc-utils';
|
||||
import { URI } from '../../../core/model/uri';
|
||||
import { BUILD_EMBEDDINGS_COMMAND } from './build-embeddings';
|
||||
|
||||
export const SHOW_SIMILAR_NOTES_COMMAND = {
|
||||
command: 'foam-vscode.show-similar-notes',
|
||||
title: 'Foam: Show Similar Notes',
|
||||
};
|
||||
|
||||
export default async function activate(
|
||||
context: vscode.ExtensionContext,
|
||||
foamPromise: Promise<Foam>
|
||||
) {
|
||||
const foam = await foamPromise;
|
||||
|
||||
context.subscriptions.push(
|
||||
vscode.commands.registerCommand(
|
||||
SHOW_SIMILAR_NOTES_COMMAND.command,
|
||||
async () => {
|
||||
await showSimilarNotes(foam);
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
async function showSimilarNotes(foam: Foam): Promise<void> {
|
||||
// Get the active editor
|
||||
const editor = vscode.window.activeTextEditor;
|
||||
if (!editor) {
|
||||
vscode.window.showInformationMessage('Please open a note first');
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the URI of the active document
|
||||
const uri = fromVsCodeUri(editor.document.uri);
|
||||
|
||||
// Check if the resource exists in workspace
|
||||
const resource = foam.workspace.find(uri);
|
||||
if (!resource) {
|
||||
vscode.window.showInformationMessage('This file is not a note');
|
||||
return;
|
||||
}
|
||||
|
||||
// Ensure embeddings are up-to-date (incremental update)
|
||||
const status: 'complete' | 'error' | 'cancelled' =
|
||||
await vscode.commands.executeCommand(BUILD_EMBEDDINGS_COMMAND.command);
|
||||
|
||||
if (status !== 'complete') {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if embedding exists for this resource
|
||||
const embedding = foam.embeddings.getEmbedding(uri);
|
||||
if (!embedding) {
|
||||
vscode.window.showInformationMessage(
|
||||
'This note hasn\'t been analyzed yet. Make sure the AI service is running and try the "Analyze Notes with AI" command.'
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Get similar notes
|
||||
const similar = foam.embeddings.getSimilar(uri, 10);
|
||||
|
||||
if (similar.length === 0) {
|
||||
vscode.window.showInformationMessage('No similar notes found');
|
||||
return;
|
||||
}
|
||||
|
||||
// Create quick pick items
|
||||
const items: vscode.QuickPickItem[] = similar.map(item => {
|
||||
const resource = foam.workspace.find(item.uri);
|
||||
const title = resource?.title || item.uri.getBasename();
|
||||
const similarityPercent = (item.similarity * 100).toFixed(1);
|
||||
|
||||
return {
|
||||
label: `$(file) ${title}`,
|
||||
description: `${similarityPercent}% similar`,
|
||||
detail: item.uri.toFsPath(),
|
||||
uri: item.uri,
|
||||
} as vscode.QuickPickItem & { uri: URI };
|
||||
});
|
||||
|
||||
// Show quick pick
|
||||
const selected = await vscode.window.showQuickPick(items, {
|
||||
placeHolder: 'Select a similar note to open',
|
||||
matchOnDescription: true,
|
||||
matchOnDetail: true,
|
||||
});
|
||||
|
||||
if (selected) {
|
||||
const selectedUri = (selected as any).uri as URI;
|
||||
const doc = await vscode.workspace.openTextDocument(
|
||||
toVsCodeUri(selectedUri)
|
||||
);
|
||||
await vscode.window.showTextDocument(doc);
|
||||
}
|
||||
}
|
||||
138
packages/foam-vscode/src/ai/vscode/panels/related-notes.ts
Normal file
138
packages/foam-vscode/src/ai/vscode/panels/related-notes.ts
Normal file
@@ -0,0 +1,138 @@
|
||||
import * as vscode from 'vscode';
|
||||
import { Foam } from '../../../core/model/foam';
|
||||
import { FoamWorkspace } from '../../../core/model/workspace';
|
||||
import { URI } from '../../../core/model/uri';
|
||||
import { fromVsCodeUri } from '../../../utils/vsc-utils';
|
||||
import { BaseTreeProvider } from '../../../features/panels/utils/base-tree-provider';
|
||||
import { ResourceTreeItem } from '../../../features/panels/utils/tree-view-utils';
|
||||
import { FoamEmbeddings } from '../../../ai/model/embeddings';
|
||||
|
||||
export default async function activate(
|
||||
context: vscode.ExtensionContext,
|
||||
foamPromise: Promise<Foam>
|
||||
) {
|
||||
const foam = await foamPromise;
|
||||
|
||||
const provider = new RelatedNotesTreeDataProvider(
|
||||
foam.workspace,
|
||||
foam.embeddings,
|
||||
context.globalState
|
||||
);
|
||||
|
||||
const treeView = vscode.window.createTreeView('foam-vscode.related-notes', {
|
||||
treeDataProvider: provider,
|
||||
showCollapseAll: false,
|
||||
});
|
||||
|
||||
const updateTreeView = async () => {
|
||||
const activeEditor = vscode.window.activeTextEditor;
|
||||
provider.target = activeEditor
|
||||
? fromVsCodeUri(activeEditor.document.uri)
|
||||
: undefined;
|
||||
await provider.refresh();
|
||||
|
||||
// Update context for conditional viewsWelcome messages
|
||||
vscode.commands.executeCommand(
|
||||
'setContext',
|
||||
'foam.relatedNotes.state',
|
||||
provider.getState()
|
||||
);
|
||||
};
|
||||
|
||||
updateTreeView();
|
||||
|
||||
context.subscriptions.push(
|
||||
provider,
|
||||
treeView,
|
||||
foam.embeddings.onDidUpdate(() => updateTreeView()),
|
||||
vscode.window.onDidChangeActiveTextEditor(() => updateTreeView()),
|
||||
provider.onDidChangeTreeData(() => {
|
||||
treeView.title = `Related Notes (${provider.nValues})`;
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
export class RelatedNotesTreeDataProvider extends BaseTreeProvider<vscode.TreeItem> {
|
||||
public target?: URI = undefined;
|
||||
public nValues = 0;
|
||||
private relatedNotes: Array<{ uri: URI; similarity: number }> = [];
|
||||
private currentNoteHasEmbedding = false;
|
||||
|
||||
constructor(
|
||||
private workspace: FoamWorkspace,
|
||||
private embeddings: FoamEmbeddings,
|
||||
public state: vscode.Memento
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
async refresh(): Promise<void> {
|
||||
const uri = this.target;
|
||||
|
||||
// Clear if no target or target is not a note
|
||||
if (!uri) {
|
||||
this.relatedNotes = [];
|
||||
this.nValues = 0;
|
||||
this.currentNoteHasEmbedding = false;
|
||||
super.refresh();
|
||||
return;
|
||||
}
|
||||
|
||||
const resource = this.workspace.find(uri);
|
||||
if (!resource || resource.type !== 'note') {
|
||||
this.relatedNotes = [];
|
||||
this.nValues = 0;
|
||||
this.currentNoteHasEmbedding = false;
|
||||
super.refresh();
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if current note has an embedding
|
||||
this.currentNoteHasEmbedding = this.embeddings.getEmbedding(uri) !== null;
|
||||
|
||||
// Get similar notes (user can click "Build Embeddings" button if needed)
|
||||
const similar = this.embeddings.getSimilar(uri, 10);
|
||||
this.relatedNotes = similar.filter(n => n.similarity > 0.6);
|
||||
this.nValues = this.relatedNotes.length;
|
||||
super.refresh();
|
||||
}
|
||||
|
||||
async getChildren(item?: vscode.TreeItem): Promise<vscode.TreeItem[]> {
|
||||
if (item) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// If no related notes found, show appropriate message in viewsWelcome
|
||||
// The empty array will trigger the viewsWelcome content
|
||||
if (this.relatedNotes.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return this.relatedNotes
|
||||
.map(({ uri, similarity }) => {
|
||||
const resource = this.workspace.find(uri);
|
||||
if (!resource) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const item = new ResourceTreeItem(resource, this.workspace);
|
||||
// Show similarity score as percentage in description
|
||||
item.description = `${Math.round(similarity * 100)}%`;
|
||||
return item;
|
||||
})
|
||||
.filter(item => item !== null) as ResourceTreeItem[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the current state of the related notes panel
|
||||
*/
|
||||
public getState(): 'no-note' | 'no-embedding' | 'ready' {
|
||||
if (!this.target) {
|
||||
return 'no-note';
|
||||
}
|
||||
if (!this.currentNoteHasEmbedding) {
|
||||
return 'no-embedding';
|
||||
}
|
||||
return 'ready';
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,10 @@ import { FoamGraph } from './graph';
|
||||
import { ResourceParser } from './note';
|
||||
import { ResourceProvider } from './provider';
|
||||
import { FoamTags } from './tags';
|
||||
import { FoamEmbeddings } from '../../ai/model/embeddings';
|
||||
import { InMemoryEmbeddingCache } from '../../ai/model/in-memory-embedding-cache';
|
||||
import { EmbeddingProvider } from '../../ai/services/embedding-provider';
|
||||
import { NoOpEmbeddingProvider } from '../../ai/services/noop-embedding-provider';
|
||||
import { Logger, withTiming, withTimingAsync } from '../utils/log';
|
||||
|
||||
export interface Services {
|
||||
@@ -18,6 +22,7 @@ export interface Foam extends IDisposable {
|
||||
workspace: FoamWorkspace;
|
||||
graph: FoamGraph;
|
||||
tags: FoamTags;
|
||||
embeddings: FoamEmbeddings;
|
||||
}
|
||||
|
||||
export const bootstrap = async (
|
||||
@@ -26,7 +31,8 @@ export const bootstrap = async (
|
||||
dataStore: IDataStore,
|
||||
parser: ResourceParser,
|
||||
initialProviders: ResourceProvider[],
|
||||
defaultExtension: string = '.md'
|
||||
defaultExtension: string = '.md',
|
||||
embeddingProvider?: EmbeddingProvider
|
||||
) => {
|
||||
const workspace = await withTimingAsync(
|
||||
() =>
|
||||
@@ -48,6 +54,22 @@ export const bootstrap = async (
|
||||
ms => Logger.info(`Tags loaded in ${ms}ms`)
|
||||
);
|
||||
|
||||
embeddingProvider = embeddingProvider ?? new NoOpEmbeddingProvider();
|
||||
const embeddings = FoamEmbeddings.fromWorkspace(
|
||||
workspace,
|
||||
embeddingProvider,
|
||||
true,
|
||||
new InMemoryEmbeddingCache()
|
||||
);
|
||||
|
||||
if (await embeddingProvider.isAvailable()) {
|
||||
Logger.info('Embeddings service initialized');
|
||||
} else {
|
||||
Logger.warn(
|
||||
'Embedding provider not available. Semantic features will be disabled.'
|
||||
);
|
||||
}
|
||||
|
||||
watcher?.onDidChange(async uri => {
|
||||
if (matcher.isMatch(uri)) {
|
||||
await workspace.fetchAndSet(uri);
|
||||
@@ -67,6 +89,7 @@ export const bootstrap = async (
|
||||
workspace,
|
||||
graph,
|
||||
tags,
|
||||
embeddings,
|
||||
services: {
|
||||
parser,
|
||||
dataStore,
|
||||
@@ -75,6 +98,7 @@ export const bootstrap = async (
|
||||
dispose: () => {
|
||||
workspace.dispose();
|
||||
graph.dispose();
|
||||
embeddings.dispose();
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
@@ -146,7 +146,6 @@ describe('Graph', () => {
|
||||
});
|
||||
const noteB = createTestNote({
|
||||
uri: '/somewhere/page-b.md',
|
||||
text: '## Section 1\n\n## Section 2',
|
||||
});
|
||||
const ws = createTestWorkspace().set(noteA).set(noteB);
|
||||
const graph = FoamGraph.fromWorkspace(ws);
|
||||
|
||||
34
packages/foam-vscode/src/core/services/progress.ts
Normal file
34
packages/foam-vscode/src/core/services/progress.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* Generic progress information for long-running operations
|
||||
*/
|
||||
export interface Progress<T = unknown> {
|
||||
/** Current item being processed (1-indexed) */
|
||||
current: number;
|
||||
/** Total number of items to process */
|
||||
total: number;
|
||||
/** Optional context data about the current item */
|
||||
context?: T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Callback for reporting progress during operations
|
||||
*/
|
||||
export type ProgressCallback<T = unknown> = (progress: Progress<T>) => void;
|
||||
|
||||
/**
|
||||
* Cancellation token for aborting long-running operations
|
||||
*/
|
||||
export interface CancellationToken {
|
||||
/** Whether cancellation has been requested */
|
||||
readonly isCancellationRequested: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Exception thrown when an operation is cancelled
|
||||
*/
|
||||
export class CancellationError extends Error {
|
||||
constructor(message: string = 'Operation cancelled') {
|
||||
super(message);
|
||||
this.name = 'CancellationError';
|
||||
}
|
||||
}
|
||||
306
packages/foam-vscode/src/core/utils/task-deduplicator.test.ts
Normal file
306
packages/foam-vscode/src/core/utils/task-deduplicator.test.ts
Normal file
@@ -0,0 +1,306 @@
|
||||
import { TaskDeduplicator } from './task-deduplicator';
|
||||
|
||||
describe('TaskDeduplicator', () => {
|
||||
describe('run', () => {
|
||||
it('should execute a task and return its result', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
const task = jest.fn(async () => 'result');
|
||||
|
||||
const result = await deduplicator.run(task);
|
||||
|
||||
expect(result).toBe('result');
|
||||
expect(task).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should deduplicate concurrent calls to the same task', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
let executeCount = 0;
|
||||
|
||||
const task = async () => {
|
||||
executeCount++;
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
return 'result';
|
||||
};
|
||||
|
||||
// Start multiple concurrent calls
|
||||
const [result1, result2, result3] = await Promise.all([
|
||||
deduplicator.run(task),
|
||||
deduplicator.run(task),
|
||||
deduplicator.run(task),
|
||||
]);
|
||||
|
||||
// All should get the same result
|
||||
expect(result1).toBe('result');
|
||||
expect(result2).toBe('result');
|
||||
expect(result3).toBe('result');
|
||||
|
||||
// Task should only execute once
|
||||
expect(executeCount).toBe(1);
|
||||
});
|
||||
|
||||
it('should call onDuplicate callback for concurrent calls', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
const onDuplicate = jest.fn();
|
||||
|
||||
const task = async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
return 'result';
|
||||
};
|
||||
|
||||
// Start concurrent calls
|
||||
const promise1 = deduplicator.run(task);
|
||||
const promise2 = deduplicator.run(task, onDuplicate);
|
||||
const promise3 = deduplicator.run(task, onDuplicate);
|
||||
|
||||
await Promise.all([promise1, promise2, promise3]);
|
||||
|
||||
// onDuplicate should be called for the 2nd and 3rd calls
|
||||
expect(onDuplicate).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should not call onDuplicate for the first call', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
const onDuplicate = jest.fn();
|
||||
const task = jest.fn(async () => 'result');
|
||||
|
||||
await deduplicator.run(task, onDuplicate);
|
||||
|
||||
expect(onDuplicate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should allow new tasks after previous task completes', async () => {
|
||||
const deduplicator = new TaskDeduplicator<number>();
|
||||
let counter = 0;
|
||||
|
||||
const task1 = async () => ++counter;
|
||||
const task2 = async () => ++counter;
|
||||
|
||||
const result1 = await deduplicator.run(task1);
|
||||
const result2 = await deduplicator.run(task2);
|
||||
|
||||
expect(result1).toBe(1);
|
||||
expect(result2).toBe(2);
|
||||
});
|
||||
|
||||
it('should propagate errors from the task', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
const error = new Error('Task failed');
|
||||
const task = jest.fn(async () => {
|
||||
throw error;
|
||||
});
|
||||
|
||||
await expect(deduplicator.run(task)).rejects.toThrow('Task failed');
|
||||
});
|
||||
|
||||
it('should propagate errors to all concurrent callers', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
const error = new Error('Task failed');
|
||||
|
||||
const task = async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
throw error;
|
||||
};
|
||||
|
||||
const promise1 = deduplicator.run(task);
|
||||
const promise2 = deduplicator.run(task);
|
||||
const promise3 = deduplicator.run(task);
|
||||
|
||||
await expect(promise1).rejects.toThrow('Task failed');
|
||||
await expect(promise2).rejects.toThrow('Task failed');
|
||||
await expect(promise3).rejects.toThrow('Task failed');
|
||||
});
|
||||
|
||||
it('should clear running task after error', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
const task1 = jest.fn(async () => {
|
||||
throw new Error('Task failed');
|
||||
});
|
||||
const task2 = jest.fn(async () => 'success');
|
||||
|
||||
// First task fails
|
||||
await expect(deduplicator.run(task1)).rejects.toThrow('Task failed');
|
||||
|
||||
// Second task should execute (not deduplicated)
|
||||
const result = await deduplicator.run(task2);
|
||||
|
||||
expect(result).toBe('success');
|
||||
expect(task1).toHaveBeenCalledTimes(1);
|
||||
expect(task2).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should handle different return types', async () => {
|
||||
// String
|
||||
const stringDeduplicator = new TaskDeduplicator<string>();
|
||||
const stringResult = await stringDeduplicator.run(async () => 'test');
|
||||
expect(stringResult).toBe('test');
|
||||
|
||||
// Number
|
||||
const numberDeduplicator = new TaskDeduplicator<number>();
|
||||
const numberResult = await numberDeduplicator.run(async () => 42);
|
||||
expect(numberResult).toBe(42);
|
||||
|
||||
// Object
|
||||
const objectDeduplicator = new TaskDeduplicator<{ value: string }>();
|
||||
const objectResult = await objectDeduplicator.run(async () => ({
|
||||
value: 'test',
|
||||
}));
|
||||
expect(objectResult).toEqual({ value: 'test' });
|
||||
|
||||
// Union types
|
||||
type Status = 'complete' | 'cancelled' | 'error';
|
||||
const statusDeduplicator = new TaskDeduplicator<Status>();
|
||||
const statusResult = await statusDeduplicator.run(
|
||||
async () => 'complete' as Status
|
||||
);
|
||||
expect(statusResult).toBe('complete');
|
||||
});
|
||||
});
|
||||
|
||||
describe('isRunning', () => {
|
||||
it('should return false when no task is running', () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
|
||||
expect(deduplicator.isRunning()).toBe(false);
|
||||
});
|
||||
|
||||
it('should return true when a task is running', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
|
||||
const task = async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
return 'result';
|
||||
};
|
||||
|
||||
const promise = deduplicator.run(task);
|
||||
|
||||
expect(deduplicator.isRunning()).toBe(true);
|
||||
|
||||
await promise;
|
||||
});
|
||||
|
||||
it('should return false after task completes', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
const task = jest.fn(async () => 'result');
|
||||
|
||||
await deduplicator.run(task);
|
||||
|
||||
expect(deduplicator.isRunning()).toBe(false);
|
||||
});
|
||||
|
||||
it('should return false after task fails', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
const task = jest.fn(async () => {
|
||||
throw new Error('Failed');
|
||||
});
|
||||
|
||||
await expect(deduplicator.run(task)).rejects.toThrow('Failed');
|
||||
|
||||
expect(deduplicator.isRunning()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('clear', () => {
|
||||
it('should clear the running task reference', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
|
||||
const task = async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
return 'result';
|
||||
};
|
||||
|
||||
const promise = deduplicator.run(task);
|
||||
|
||||
expect(deduplicator.isRunning()).toBe(true);
|
||||
|
||||
deduplicator.clear();
|
||||
|
||||
expect(deduplicator.isRunning()).toBe(false);
|
||||
|
||||
// Original promise should still complete
|
||||
await expect(promise).resolves.toBe('result');
|
||||
});
|
||||
|
||||
it('should allow new task after manual clear', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
let executeCount = 0;
|
||||
|
||||
const task = async () => {
|
||||
executeCount++;
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
return 'result';
|
||||
};
|
||||
|
||||
// Start first task
|
||||
const promise1 = deduplicator.run(task);
|
||||
|
||||
// Clear while still running
|
||||
deduplicator.clear();
|
||||
|
||||
// Start second task (should not be deduplicated)
|
||||
const promise2 = deduplicator.run(task);
|
||||
|
||||
await Promise.all([promise1, promise2]);
|
||||
|
||||
// Both tasks should have executed
|
||||
expect(executeCount).toBe(2);
|
||||
});
|
||||
|
||||
it('should be safe to call when no task is running', () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
|
||||
expect(() => deduplicator.clear()).not.toThrow();
|
||||
expect(deduplicator.isRunning()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('edge cases', () => {
|
||||
it('should handle tasks that resolve immediately', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
const task = jest.fn(async () => 'immediate');
|
||||
|
||||
const result = await deduplicator.run(task);
|
||||
|
||||
expect(result).toBe('immediate');
|
||||
expect(deduplicator.isRunning()).toBe(false);
|
||||
});
|
||||
|
||||
it('should handle tasks that throw synchronously', async () => {
|
||||
const deduplicator = new TaskDeduplicator<string>();
|
||||
const task = jest.fn(() => {
|
||||
throw new Error('Sync error');
|
||||
});
|
||||
|
||||
await expect(deduplicator.run(task as any)).rejects.toThrow('Sync error');
|
||||
expect(deduplicator.isRunning()).toBe(false);
|
||||
});
|
||||
|
||||
it('should handle null/undefined results', async () => {
|
||||
const nullDeduplicator = new TaskDeduplicator<null>();
|
||||
const nullResult = await nullDeduplicator.run(async () => null);
|
||||
expect(nullResult).toBeNull();
|
||||
|
||||
const undefinedDeduplicator = new TaskDeduplicator<undefined>();
|
||||
const undefinedResult = await undefinedDeduplicator.run(
|
||||
async () => undefined
|
||||
);
|
||||
expect(undefinedResult).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should handle sequential calls with delays between them', async () => {
|
||||
const deduplicator = new TaskDeduplicator<number>();
|
||||
let counter = 0;
|
||||
|
||||
const task = async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
return ++counter;
|
||||
};
|
||||
|
||||
const result1 = await deduplicator.run(task);
|
||||
await new Promise(resolve => setTimeout(resolve, 20));
|
||||
const result2 = await deduplicator.run(task);
|
||||
|
||||
expect(result1).toBe(1);
|
||||
expect(result2).toBe(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
67
packages/foam-vscode/src/core/utils/task-deduplicator.ts
Normal file
67
packages/foam-vscode/src/core/utils/task-deduplicator.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* A utility class for deduplicating concurrent async operations.
|
||||
* When multiple calls are made while a task is running, subsequent calls
|
||||
* will wait for and receive the result of the already-running task instead
|
||||
* of starting a new one.
|
||||
*
|
||||
* @example
|
||||
* const deduplicator = new TaskDeduplicator<string>();
|
||||
*
|
||||
* async function expensiveOperation(input: string): Promise<string> {
|
||||
* return deduplicator.run(async () => {
|
||||
* // Expensive work here
|
||||
* return result;
|
||||
* });
|
||||
* }
|
||||
*
|
||||
* // Multiple concurrent calls will share the same execution
|
||||
* const [result1, result2] = await Promise.all([
|
||||
* expensiveOperation("test"),
|
||||
* expensiveOperation("test"),
|
||||
* ]);
|
||||
* // Only runs once, both get the same result
|
||||
*/
|
||||
export class TaskDeduplicator<T> {
|
||||
private runningTask: Promise<T> | null = null;
|
||||
|
||||
/**
|
||||
* Run a task with deduplication.
|
||||
* If a task is already running, waits for it to complete and returns its result.
|
||||
* Otherwise, starts the task and stores its promise for other callers to await.
|
||||
*
|
||||
* @param task The async function to execute
|
||||
* @param onDuplicate Optional callback when a duplicate call is detected
|
||||
* @returns The result of the task
|
||||
*/
|
||||
async run(task: () => Promise<T>, onDuplicate?: () => void): Promise<T> {
|
||||
// If already running, wait for the existing task
|
||||
if (this.runningTask) {
|
||||
onDuplicate?.();
|
||||
return await this.runningTask;
|
||||
}
|
||||
|
||||
// Start the task and store the promise
|
||||
this.runningTask = task();
|
||||
|
||||
try {
|
||||
return await this.runningTask;
|
||||
} finally {
|
||||
// Clear the task when done
|
||||
this.runningTask = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a task is currently running
|
||||
*/
|
||||
isRunning(): boolean {
|
||||
return this.runningTask !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the running task reference (useful for testing or error recovery)
|
||||
*/
|
||||
clear(): void {
|
||||
this.runningTask = null;
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import { VsCodeWatcher } from './services/watcher';
|
||||
import { createMarkdownParser } from './core/services/markdown-parser';
|
||||
import VsCodeBasedParserCache from './services/cache';
|
||||
import { createMatcherAndDataStore } from './services/editor';
|
||||
import { OllamaEmbeddingProvider } from './ai/providers/ollama/ollama-provider';
|
||||
|
||||
export async function activate(context: ExtensionContext) {
|
||||
const logger = new VsCodeOutputLogger();
|
||||
@@ -71,13 +72,20 @@ export async function activate(context: ExtensionContext) {
|
||||
attachmentExtConfig
|
||||
);
|
||||
|
||||
// Initialize embedding provider
|
||||
const aiEnabled = workspace.getConfiguration('foam.experimental').get('ai');
|
||||
const embeddingProvider = aiEnabled
|
||||
? new OllamaEmbeddingProvider()
|
||||
: undefined;
|
||||
|
||||
const foamPromise = bootstrap(
|
||||
matcher,
|
||||
watcher,
|
||||
dataStore,
|
||||
parser,
|
||||
[markdownProvider, attachmentProvider],
|
||||
defaultExtension
|
||||
defaultExtension,
|
||||
embeddingProvider
|
||||
);
|
||||
|
||||
// Load the features
|
||||
|
||||
@@ -13,3 +13,5 @@ export { default as createNote } from './create-note';
|
||||
export { default as searchTagCommand } from './search-tag';
|
||||
export { default as renameTagCommand } from './rename-tag';
|
||||
export { default as convertLinksCommand } from './convert-links';
|
||||
export { default as showSimilarNotesCommand } from '../../ai/vscode/commands/show-similar-notes';
|
||||
export { default as buildEmbeddingsCommand } from '../../ai/vscode/commands/build-embeddings';
|
||||
|
||||
@@ -4,3 +4,4 @@ export { default as orphans } from './orphans';
|
||||
export { default as placeholders } from './placeholders';
|
||||
export { default as tags } from './tags-explorer';
|
||||
export { default as notes } from './notes-explorer';
|
||||
export { default as relatedNotes } from '../../ai/vscode/panels/related-notes';
|
||||
|
||||
@@ -80,7 +80,9 @@ export const waitForNoteInFoamWorkspace = async (uri: URI, timeout = 5000) => {
|
||||
}
|
||||
await wait(100);
|
||||
}
|
||||
return false;
|
||||
throw new Error(
|
||||
`Timeout waiting for note ${uri.toString()} in Foam workspace`
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -9,11 +9,49 @@ import { FoamWorkspace } from '../core/model/workspace';
|
||||
import { MarkdownResourceProvider } from '../core/services/markdown-provider';
|
||||
import { Resource } from '../core/model/note';
|
||||
import { createMarkdownParser } from '../core/services/markdown-parser';
|
||||
import { IDataStore } from '../core/services/datastore';
|
||||
|
||||
export { default as waitForExpect } from 'wait-for-expect';
|
||||
|
||||
Logger.setLevel('error');
|
||||
|
||||
/**
|
||||
* An in-memory data store for testing that stores file content in a Map.
|
||||
* This allows tests to provide text content for notes without touching the filesystem.
|
||||
*/
|
||||
export class InMemoryDataStore implements IDataStore {
|
||||
private files = new Map<string, string>();
|
||||
|
||||
/**
|
||||
* Set the content for a file
|
||||
*/
|
||||
set(uri: URI, content: string): void {
|
||||
this.files.set(uri.path, content);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a file
|
||||
*/
|
||||
delete(uri: URI): void {
|
||||
this.files.delete(uri.path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all files
|
||||
*/
|
||||
clear(): void {
|
||||
this.files.clear();
|
||||
}
|
||||
|
||||
async list(): Promise<URI[]> {
|
||||
return Array.from(this.files.keys()).map(path => URI.parse(path, 'file'));
|
||||
}
|
||||
|
||||
async read(uri: URI): Promise<string | null> {
|
||||
return this.files.get(uri.path) ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
export const TEST_DATA_DIR = URI.file(__dirname).joinPath(
|
||||
'..',
|
||||
'..',
|
||||
@@ -29,11 +67,14 @@ const position = Range.create(0, 0, 0, 100);
|
||||
*/
|
||||
export const strToUri = URI.file;
|
||||
|
||||
export const createTestWorkspace = (workspaceRoots: URI[] = []) => {
|
||||
export const createTestWorkspace = (
|
||||
workspaceRoots: URI[] = [],
|
||||
dataStore?: IDataStore
|
||||
) => {
|
||||
const workspace = new FoamWorkspace();
|
||||
const parser = createMarkdownParser();
|
||||
const provider = new MarkdownResourceProvider(
|
||||
{
|
||||
dataStore ?? {
|
||||
read: _ => Promise.resolve(''),
|
||||
list: () => Promise.resolve([]),
|
||||
},
|
||||
@@ -51,7 +92,6 @@ export const createTestNote = (params: {
|
||||
links?: Array<{ slug: string; definitionUrl?: string } | { to: string }>;
|
||||
tags?: string[];
|
||||
aliases?: string[];
|
||||
text?: string;
|
||||
sections?: string[];
|
||||
root?: URI;
|
||||
type?: string;
|
||||
|
||||
@@ -17,10 +17,12 @@ import { createMarkdownParser } from '../core/services/markdown-parser';
|
||||
import {
|
||||
GenericDataStore,
|
||||
AlwaysIncludeMatcher,
|
||||
IWatcher,
|
||||
} from '../core/services/datastore';
|
||||
import { MarkdownResourceProvider } from '../core/services/markdown-provider';
|
||||
import { randomString } from './test-utils';
|
||||
import micromatch from 'micromatch';
|
||||
import { Emitter } from '../core/common/event';
|
||||
|
||||
interface Thenable<T> {
|
||||
then<TResult>(
|
||||
@@ -247,6 +249,13 @@ export function createVSCodeUri(foamUri: URI): Uri {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert VS Code Uri to Foam URI
|
||||
*/
|
||||
export function fromVsCodeUri(vsCodeUri: Uri): URI {
|
||||
return URI.file(vsCodeUri.fsPath);
|
||||
}
|
||||
|
||||
// VS Code Uri static methods
|
||||
// eslint-disable-next-line @typescript-eslint/no-redeclare
|
||||
export const Uri = {
|
||||
@@ -423,6 +432,12 @@ export enum DiagnosticSeverity {
|
||||
Hint = 3,
|
||||
}
|
||||
|
||||
export enum ProgressLocation {
|
||||
SourceControl = 1,
|
||||
Window = 10,
|
||||
Notification = 15,
|
||||
}
|
||||
|
||||
// ===== Code Actions =====
|
||||
|
||||
export class CodeActionKind {
|
||||
@@ -588,6 +603,57 @@ export interface Disposable {
|
||||
dispose(): void;
|
||||
}
|
||||
|
||||
// ===== Cancellation =====
|
||||
|
||||
export interface CancellationToken {
|
||||
readonly isCancellationRequested: boolean;
|
||||
readonly onCancellationRequested: Event<any>;
|
||||
}
|
||||
|
||||
export class CancellationTokenSource {
|
||||
private _token: CancellationToken | undefined;
|
||||
private _emitter: EventEmitter<any> | undefined;
|
||||
private _isCancelled = false;
|
||||
|
||||
get token(): CancellationToken {
|
||||
if (!this._token) {
|
||||
this._emitter = new EventEmitter<any>();
|
||||
this._token = {
|
||||
isCancellationRequested: this._isCancelled,
|
||||
onCancellationRequested: this._emitter.event,
|
||||
};
|
||||
}
|
||||
return this._token;
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
if (!this._isCancelled) {
|
||||
this._isCancelled = true;
|
||||
if (this._emitter) {
|
||||
this._emitter.fire(undefined);
|
||||
}
|
||||
// Update token state
|
||||
if (this._token) {
|
||||
(this._token as any).isCancellationRequested = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
if (this._emitter) {
|
||||
this._emitter.dispose();
|
||||
this._emitter = undefined;
|
||||
}
|
||||
this._token = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Progress =====
|
||||
|
||||
export interface Progress<T> {
|
||||
report(value: T): void;
|
||||
}
|
||||
|
||||
export class EventEmitter<T> {
|
||||
private listeners: ((e: T) => any)[] = [];
|
||||
|
||||
@@ -791,11 +857,21 @@ class MockTextDocument implements TextDocument {
|
||||
this._content = content;
|
||||
// Write the content to file if provided
|
||||
try {
|
||||
const existed = fs.existsSync(uri.fsPath);
|
||||
const dir = path.dirname(uri.fsPath);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(uri.fsPath, content);
|
||||
|
||||
// Manually fire watcher events (can't use async workspace.fs in constructor)
|
||||
for (const watcher of mockState.fileWatchers) {
|
||||
if (existed) {
|
||||
watcher._fireChange(uri);
|
||||
} else {
|
||||
watcher._fireCreate(uri);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Ignore write errors in mock
|
||||
}
|
||||
@@ -1115,10 +1191,31 @@ class MockFileSystem implements FileSystem {
|
||||
}
|
||||
|
||||
async writeFile(uri: Uri, content: Uint8Array): Promise<void> {
|
||||
// Check if file exists before writing
|
||||
const existed = await this.exists(uri);
|
||||
|
||||
// Ensure directory exists
|
||||
const dir = path.dirname(uri.fsPath);
|
||||
await fs.promises.mkdir(dir, { recursive: true });
|
||||
await fs.promises.writeFile(uri.fsPath, content);
|
||||
|
||||
// Fire watcher events
|
||||
for (const watcher of mockState.fileWatchers) {
|
||||
if (existed) {
|
||||
watcher._fireChange(uri);
|
||||
} else {
|
||||
watcher._fireCreate(uri);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async exists(uri: Uri): Promise<boolean> {
|
||||
try {
|
||||
await fs.promises.access(uri.fsPath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async delete(uri: Uri, options?: { recursive?: boolean }): Promise<void> {
|
||||
@@ -1133,6 +1230,11 @@ class MockFileSystem implements FileSystem {
|
||||
} else {
|
||||
await fs.promises.unlink(uri.fsPath);
|
||||
}
|
||||
|
||||
// Fire watcher events
|
||||
for (const watcher of mockState.fileWatchers) {
|
||||
watcher._fireDelete(uri);
|
||||
}
|
||||
}
|
||||
|
||||
async stat(
|
||||
@@ -1175,6 +1277,84 @@ class MockFileSystem implements FileSystem {
|
||||
options?: { overwrite?: boolean }
|
||||
): Promise<void> {
|
||||
await fs.promises.rename(source.fsPath, target.fsPath);
|
||||
|
||||
// Fire watcher events (rename = delete + create)
|
||||
for (const watcher of mockState.fileWatchers) {
|
||||
watcher._fireDelete(source);
|
||||
watcher._fireCreate(target);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== File System Watcher =====
|
||||
|
||||
export interface FileSystemWatcher extends Disposable {
|
||||
onDidCreate: Event<Uri>;
|
||||
onDidChange: Event<Uri>;
|
||||
onDidDelete: Event<Uri>;
|
||||
ignoreCreateEvents: boolean;
|
||||
ignoreChangeEvents: boolean;
|
||||
ignoreDeleteEvents: boolean;
|
||||
}
|
||||
|
||||
class MockFileSystemWatcher implements FileSystemWatcher {
|
||||
private onDidCreateEmitter = new Emitter<Uri>();
|
||||
private onDidChangeEmitter = new Emitter<Uri>();
|
||||
private onDidDeleteEmitter = new Emitter<Uri>();
|
||||
|
||||
onDidCreate = this.onDidCreateEmitter.event;
|
||||
onDidChange = this.onDidChangeEmitter.event;
|
||||
onDidDelete = this.onDidDeleteEmitter.event;
|
||||
|
||||
ignoreCreateEvents = false;
|
||||
ignoreChangeEvents = false;
|
||||
ignoreDeleteEvents = false;
|
||||
|
||||
constructor(private pattern: string) {
|
||||
// Register this watcher in mockState (will be added to mockState)
|
||||
if (mockState.fileWatchers) {
|
||||
mockState.fileWatchers.push(this);
|
||||
}
|
||||
}
|
||||
|
||||
// Internal methods called by MockFileSystem
|
||||
_fireCreate(uri: Uri) {
|
||||
if (!this.ignoreCreateEvents && this.matches(uri)) {
|
||||
this.onDidCreateEmitter.fire(uri);
|
||||
}
|
||||
}
|
||||
|
||||
_fireChange(uri: Uri) {
|
||||
if (!this.ignoreChangeEvents && this.matches(uri)) {
|
||||
this.onDidChangeEmitter.fire(uri);
|
||||
}
|
||||
}
|
||||
|
||||
_fireDelete(uri: Uri) {
|
||||
if (!this.ignoreDeleteEvents && this.matches(uri)) {
|
||||
this.onDidDeleteEmitter.fire(uri);
|
||||
}
|
||||
}
|
||||
|
||||
private matches(uri: Uri): boolean {
|
||||
const workspaceFolder = mockState.workspaceFolders[0];
|
||||
if (!workspaceFolder) return false;
|
||||
|
||||
const relativePath = path.relative(workspaceFolder.uri.fsPath, uri.fsPath);
|
||||
// Use micromatch (already imported) for glob matching
|
||||
return micromatch.isMatch(relativePath, this.pattern);
|
||||
}
|
||||
|
||||
dispose() {
|
||||
if (mockState.fileWatchers) {
|
||||
const index = mockState.fileWatchers.indexOf(this);
|
||||
if (index >= 0) {
|
||||
mockState.fileWatchers.splice(index, 1);
|
||||
}
|
||||
}
|
||||
this.onDidCreateEmitter.dispose();
|
||||
this.onDidChangeEmitter.dispose();
|
||||
this.onDidDeleteEmitter.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1343,17 +1523,41 @@ class TestFoam {
|
||||
// Create resource providers
|
||||
const providers = [new MarkdownResourceProvider(dataStore, parser)];
|
||||
|
||||
// Use the bootstrap function without file watcher (simpler for tests)
|
||||
// Create file watcher for automatic workspace updates
|
||||
const vsCodeWatcher = workspace.createFileSystemWatcher('**/*');
|
||||
|
||||
// Convert VS Code Uri events to Foam URI events
|
||||
const onDidCreateEmitter = new Emitter<URI>();
|
||||
const onDidChangeEmitter = new Emitter<URI>();
|
||||
const onDidDeleteEmitter = new Emitter<URI>();
|
||||
|
||||
vsCodeWatcher.onDidCreate(uri =>
|
||||
onDidCreateEmitter.fire(fromVsCodeUri(uri))
|
||||
);
|
||||
vsCodeWatcher.onDidChange(uri =>
|
||||
onDidChangeEmitter.fire(fromVsCodeUri(uri))
|
||||
);
|
||||
vsCodeWatcher.onDidDelete(uri =>
|
||||
onDidDeleteEmitter.fire(fromVsCodeUri(uri))
|
||||
);
|
||||
|
||||
const foamWatcher: IWatcher = {
|
||||
onDidCreate: onDidCreateEmitter.event,
|
||||
onDidChange: onDidChangeEmitter.event,
|
||||
onDidDelete: onDidDeleteEmitter.event,
|
||||
};
|
||||
|
||||
// Use the bootstrap function with file watcher
|
||||
const foam = await bootstrap(
|
||||
matcher,
|
||||
undefined,
|
||||
foamWatcher,
|
||||
dataStore,
|
||||
parser,
|
||||
providers,
|
||||
'.md'
|
||||
);
|
||||
|
||||
Logger.info('Mock Foam instance created (manual reload for tests)');
|
||||
Logger.info('Mock Foam instance created with file watcher');
|
||||
return foam;
|
||||
}
|
||||
|
||||
@@ -1399,13 +1603,14 @@ async function initializeFoamCommands(foam: Foam): Promise<void> {
|
||||
await foamCommands.updateWikilinksCommand(mockContext, foamPromise);
|
||||
await foamCommands.openDailyNoteForDateCommand(mockContext, foamPromise);
|
||||
await foamCommands.convertLinksCommand(mockContext, foamPromise);
|
||||
await foamCommands.buildEmbeddingsCommand(mockContext, foamPromise);
|
||||
await foamCommands.openDailyNoteCommand(mockContext, foamPromise);
|
||||
await foamCommands.openDatedNote(mockContext, foamPromise);
|
||||
|
||||
// Commands that only need context
|
||||
await foamCommands.copyWithoutBracketsCommand(mockContext);
|
||||
await foamCommands.createFromTemplateCommand(mockContext);
|
||||
await foamCommands.createNewTemplate(mockContext);
|
||||
await foamCommands.openDailyNoteCommand(mockContext, foamPromise);
|
||||
await foamCommands.openDatedNote(mockContext, foamPromise);
|
||||
|
||||
Logger.info('Foam commands initialized successfully in mock environment');
|
||||
}
|
||||
@@ -1420,6 +1625,7 @@ const mockState = {
|
||||
commands: new Map<string, (...args: any[]) => any>(),
|
||||
fileSystem: new MockFileSystem(),
|
||||
configuration: new MockWorkspaceConfiguration(),
|
||||
fileWatchers: [] as MockFileSystemWatcher[],
|
||||
};
|
||||
|
||||
// Window namespace
|
||||
@@ -1514,6 +1720,31 @@ export const window = {
|
||||
message
|
||||
);
|
||||
},
|
||||
|
||||
async withProgress<R>(
|
||||
options: {
|
||||
location: ProgressLocation;
|
||||
title?: string;
|
||||
cancellable?: boolean;
|
||||
},
|
||||
task: (
|
||||
progress: Progress<{ message?: string; increment?: number }>,
|
||||
token: CancellationToken
|
||||
) => Thenable<R>
|
||||
): Promise<R> {
|
||||
const tokenSource = new CancellationTokenSource();
|
||||
const progress: Progress<{ message?: string; increment?: number }> = {
|
||||
report: () => {
|
||||
// No-op in mock, but can be overridden in tests
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
return await task(progress, tokenSource.token);
|
||||
} finally {
|
||||
tokenSource.dispose();
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
// Workspace namespace
|
||||
@@ -1528,6 +1759,10 @@ export const workspace = {
|
||||
return mockState.fileSystem;
|
||||
},
|
||||
|
||||
createFileSystemWatcher(globPattern: string): FileSystemWatcher {
|
||||
return new MockFileSystemWatcher(globPattern);
|
||||
},
|
||||
|
||||
getConfiguration(section?: string): WorkspaceConfiguration {
|
||||
if (section) {
|
||||
// Return a scoped configuration for the specific section
|
||||
|
||||
Reference in New Issue
Block a user