mirror of
https://github.com/danielmiessler/Fabric.git
synced 2026-01-07 21:44:02 -05:00
feat: upgrade pdfjs and add SSR-safe dynamic PDF worker init
- Upgrade `pdfjs-dist` to v5 with new engine requirement - Dynamically import PDF.js to avoid SSR import-time crashes - Configure PDF worker via CDN using runtime PDF.js version - Update PDF conversion pipeline to use lazy initialization - Guard chat message localStorage persistence behind browser checks - Reformat ChatService with consistent imports and typings - Bump `patch-package` and refresh pnpm lock dependency graph - Add `skeletonlabs` to VSCode spellcheck dictionary
This commit is contained in:
Kayvan Sylvan
1
.vscode/settings.json
vendored
1
.vscode/settings.json
vendored
@@ -166,6 +166,7 @@
|
||||
"sess",
|
||||
"sgaunet",
|
||||
"shellquote",
|
||||
"skeletonlabs",
|
||||
"SSEHTTP",
|
||||
"storer",
|
||||
"Streamlit",
|
||||
|
||||
@@ -26,9 +26,9 @@
|
||||
"eslint-plugin-svelte": "^2.46.1",
|
||||
"lucide-svelte": "^0.309.0",
|
||||
"mdsvex": "^0.11.2",
|
||||
"patch-package": "^8.0.0",
|
||||
"patch-package": "^8.0.1",
|
||||
"pdf-to-markdown-core": "github:jzillmann/pdf-to-markdown#modularize",
|
||||
"pdfjs-dist": "^4.2.67",
|
||||
"pdfjs-dist": "^5.4.449",
|
||||
"postcss": "^8.5.3",
|
||||
"postcss-load-config": "^6.0.1",
|
||||
"rehype-autolink-headings": "^7.1.0",
|
||||
|
||||
549
web/pnpm-lock.yaml
generated
549
web/pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -1,239 +1,284 @@
|
||||
import { get } from "svelte/store";
|
||||
import type {
|
||||
ChatRequest,
|
||||
StreamResponse,
|
||||
ChatError as IChatError,
|
||||
ChatPrompt
|
||||
} from '$lib/interfaces/chat-interface';
|
||||
import { get } from 'svelte/store';
|
||||
import { modelConfig } from '$lib/store/model-store';
|
||||
import { systemPrompt, selectedPatternName, patternVariables } from '$lib/store/pattern-store';
|
||||
import { chatConfig } from '$lib/store/chat-config';
|
||||
import { messageStore } from '$lib/store/chat-store';
|
||||
import { languageStore } from '$lib/store/language-store';
|
||||
import { selectedStrategy } from '$lib/store/strategy-store';
|
||||
ChatPrompt,
|
||||
ChatRequest,
|
||||
ChatError as IChatError,
|
||||
StreamResponse,
|
||||
} from "$lib/interfaces/chat-interface";
|
||||
import { chatConfig } from "$lib/store/chat-config";
|
||||
import { languageStore } from "$lib/store/language-store";
|
||||
import { modelConfig } from "$lib/store/model-store";
|
||||
import {
|
||||
patternVariables,
|
||||
selectedPatternName,
|
||||
systemPrompt,
|
||||
} from "$lib/store/pattern-store";
|
||||
import { selectedStrategy } from "$lib/store/strategy-store";
|
||||
|
||||
class LanguageValidator {
|
||||
constructor(private targetLanguage: string) {}
|
||||
constructor(private targetLanguage: string) {}
|
||||
|
||||
enforceLanguage(content: string): string {
|
||||
if (this.targetLanguage === 'en') return content;
|
||||
return `[Language: ${this.targetLanguage}]\n${content}`;
|
||||
}
|
||||
enforceLanguage(content: string): string {
|
||||
if (this.targetLanguage === "en") return content;
|
||||
return `[Language: ${this.targetLanguage}]\n${content}`;
|
||||
}
|
||||
}
|
||||
|
||||
export class ChatError extends Error implements IChatError {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly code: string = 'CHAT_ERROR',
|
||||
public readonly details?: unknown
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ChatError';
|
||||
}
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly code: string = "CHAT_ERROR",
|
||||
public readonly details?: unknown,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "ChatError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ChatService {
|
||||
private validator: LanguageValidator;
|
||||
private validator: LanguageValidator;
|
||||
|
||||
constructor() {
|
||||
this.validator = new LanguageValidator(get(languageStore));
|
||||
}
|
||||
constructor() {
|
||||
this.validator = new LanguageValidator(get(languageStore));
|
||||
}
|
||||
|
||||
private async fetchStream(request: ChatRequest): Promise<ReadableStream<StreamResponse>> {
|
||||
try {
|
||||
console.log('\n=== ChatService Request Start ===');
|
||||
console.log('1. Request details:', {
|
||||
language: get(languageStore),
|
||||
pattern: get(selectedPatternName),
|
||||
promptCount: request.prompts?.length,
|
||||
messageCount: request.messages?.length
|
||||
});
|
||||
// NEW: Log the full payload before sending to backend
|
||||
console.log('Final ChatRequest payload:', JSON.stringify(request, null, 2));
|
||||
private async fetchStream(
|
||||
request: ChatRequest,
|
||||
): Promise<ReadableStream<StreamResponse>> {
|
||||
try {
|
||||
console.log("\n=== ChatService Request Start ===");
|
||||
console.log("1. Request details:", {
|
||||
language: get(languageStore),
|
||||
pattern: get(selectedPatternName),
|
||||
promptCount: request.prompts?.length,
|
||||
messageCount: request.messages?.length,
|
||||
});
|
||||
// NEW: Log the full payload before sending to backend
|
||||
console.log(
|
||||
"Final ChatRequest payload:",
|
||||
JSON.stringify(request, null, 2),
|
||||
);
|
||||
|
||||
const response = await fetch('/api/chat', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(request),
|
||||
});
|
||||
const response = await fetch("/api/chat", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(request),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ChatError(`HTTP error! status: ${response.status}`, 'HTTP_ERROR', { status: response.status });
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new ChatError(
|
||||
`HTTP error! status: ${response.status}`,
|
||||
"HTTP_ERROR",
|
||||
{ status: response.status },
|
||||
);
|
||||
}
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) {
|
||||
throw new ChatError('Response body is null', 'NULL_RESPONSE');
|
||||
}
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) {
|
||||
throw new ChatError("Response body is null", "NULL_RESPONSE");
|
||||
}
|
||||
|
||||
return this.createMessageStream(reader);
|
||||
} catch (error) {
|
||||
if (error instanceof ChatError) throw error;
|
||||
throw new ChatError('Failed to fetch chat stream', 'FETCH_ERROR', error);
|
||||
}
|
||||
}
|
||||
return this.createMessageStream(reader);
|
||||
} catch (error) {
|
||||
if (error instanceof ChatError) throw error;
|
||||
throw new ChatError("Failed to fetch chat stream", "FETCH_ERROR", error);
|
||||
}
|
||||
}
|
||||
|
||||
private cleanPatternOutput(content: string): string {
|
||||
// Remove markdown fence if present
|
||||
let cleaned = content.replace(/^```markdown\n/, '');
|
||||
cleaned = cleaned.replace(/\n```$/, '');
|
||||
private cleanPatternOutput(content: string): string {
|
||||
// Remove markdown fence if present
|
||||
let cleaned = content.replace(/^```markdown\n/, "");
|
||||
cleaned = cleaned.replace(/\n```$/, "");
|
||||
|
||||
// Existing cleaning
|
||||
cleaned = cleaned.replace(/^# OUTPUT\s*\n/, '');
|
||||
cleaned = cleaned.replace(/^\s*\n/, '');
|
||||
cleaned = cleaned.replace(/\n\s*$/, '');
|
||||
cleaned = cleaned.replace(/^#\s+([A-Z]+):/gm, '$1:');
|
||||
cleaned = cleaned.replace(/^#\s+([A-Z]+)\s*$/gm, '$1');
|
||||
cleaned = cleaned.trim();
|
||||
cleaned = cleaned.replace(/\n{3,}/g, '\n\n');
|
||||
return cleaned;
|
||||
}
|
||||
// Existing cleaning
|
||||
cleaned = cleaned.replace(/^# OUTPUT\s*\n/, "");
|
||||
cleaned = cleaned.replace(/^\s*\n/, "");
|
||||
cleaned = cleaned.replace(/\n\s*$/, "");
|
||||
cleaned = cleaned.replace(/^#\s+([A-Z]+):/gm, "$1:");
|
||||
cleaned = cleaned.replace(/^#\s+([A-Z]+)\s*$/gm, "$1");
|
||||
cleaned = cleaned.trim();
|
||||
cleaned = cleaned.replace(/\n{3,}/g, "\n\n");
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
private createMessageStream(reader: ReadableStreamDefaultReader<Uint8Array>): ReadableStream<StreamResponse> {
|
||||
let buffer = '';
|
||||
const cleanPatternOutput = this.cleanPatternOutput.bind(this);
|
||||
const language = get(languageStore);
|
||||
const validator = new LanguageValidator(language);
|
||||
private createMessageStream(
|
||||
reader: ReadableStreamDefaultReader<Uint8Array>,
|
||||
): ReadableStream<StreamResponse> {
|
||||
let buffer = "";
|
||||
const cleanPatternOutput = this.cleanPatternOutput.bind(this);
|
||||
const language = get(languageStore);
|
||||
const validator = new LanguageValidator(language);
|
||||
|
||||
const processResponse = (response: StreamResponse) => {
|
||||
const pattern = get(selectedPatternName);
|
||||
const processResponse = (response: StreamResponse) => {
|
||||
const pattern = get(selectedPatternName);
|
||||
|
||||
if (pattern) {
|
||||
response.content = cleanPatternOutput(response.content);
|
||||
// Simplified format determination - always markdown unless mermaid
|
||||
const isMermaid = [
|
||||
'graph TD', 'gantt', 'flowchart',
|
||||
'sequenceDiagram', 'classDiagram', 'stateDiagram'
|
||||
].some(starter => response.content.trim().startsWith(starter));
|
||||
if (pattern) {
|
||||
response.content = cleanPatternOutput(response.content);
|
||||
// Simplified format determination - always markdown unless mermaid
|
||||
const isMermaid = [
|
||||
"graph TD",
|
||||
"gantt",
|
||||
"flowchart",
|
||||
"sequenceDiagram",
|
||||
"classDiagram",
|
||||
"stateDiagram",
|
||||
].some((starter) => response.content.trim().startsWith(starter));
|
||||
|
||||
response.format = isMermaid ? 'mermaid' : 'markdown';
|
||||
}
|
||||
response.format = isMermaid ? "mermaid" : "markdown";
|
||||
}
|
||||
|
||||
if (response.type === 'content') {
|
||||
response.content = validator.enforceLanguage(response.content);
|
||||
}
|
||||
if (response.type === "content") {
|
||||
response.content = validator.enforceLanguage(response.content);
|
||||
}
|
||||
|
||||
return response;
|
||||
};
|
||||
return new ReadableStream({
|
||||
async start(controller) {
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
return response;
|
||||
};
|
||||
return new ReadableStream({
|
||||
async start(controller) {
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += new TextDecoder().decode(value);
|
||||
const messages = buffer.split('\n\n').filter(msg => msg.startsWith('data: '));
|
||||
buffer += new TextDecoder().decode(value);
|
||||
const messages = buffer
|
||||
.split("\n\n")
|
||||
.filter((msg) => msg.startsWith("data: "));
|
||||
|
||||
if (messages.length > 1) {
|
||||
buffer = messages.pop() || '';
|
||||
for (const msg of messages) {
|
||||
try {
|
||||
let response = JSON.parse(msg.slice(6)) as StreamResponse;
|
||||
response = processResponse(response);
|
||||
controller.enqueue(response);
|
||||
} catch (parseError) {
|
||||
console.error('Error parsing stream message:', parseError);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (messages.length > 1) {
|
||||
buffer = messages.pop() || "";
|
||||
for (const msg of messages) {
|
||||
try {
|
||||
let response = JSON.parse(msg.slice(6)) as StreamResponse;
|
||||
response = processResponse(response);
|
||||
controller.enqueue(response);
|
||||
} catch (parseError) {
|
||||
console.error("Error parsing stream message:", parseError);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (buffer.startsWith('data: ')) {
|
||||
try {
|
||||
let response = JSON.parse(buffer.slice(6)) as StreamResponse;
|
||||
response = processResponse(response);
|
||||
controller.enqueue(response);
|
||||
} catch (parseError) {
|
||||
console.error('Error parsing final message:', parseError);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
controller.error(new ChatError('Stream processing error', 'STREAM_ERROR', error));
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
controller.close();
|
||||
}
|
||||
},
|
||||
cancel() {
|
||||
reader.cancel();
|
||||
}
|
||||
});
|
||||
}
|
||||
if (buffer.startsWith("data: ")) {
|
||||
try {
|
||||
let response = JSON.parse(buffer.slice(6)) as StreamResponse;
|
||||
response = processResponse(response);
|
||||
controller.enqueue(response);
|
||||
} catch (parseError) {
|
||||
console.error("Error parsing final message:", parseError);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
controller.error(
|
||||
new ChatError("Stream processing error", "STREAM_ERROR", error),
|
||||
);
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
controller.close();
|
||||
}
|
||||
},
|
||||
cancel() {
|
||||
reader.cancel();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private createChatPrompt(userInput: string, systemPromptText?: string): ChatPrompt {
|
||||
const config = get(modelConfig);
|
||||
const language = get(languageStore);
|
||||
private createChatPrompt(
|
||||
userInput: string,
|
||||
systemPromptText?: string,
|
||||
): ChatPrompt {
|
||||
const config = get(modelConfig);
|
||||
const language = get(languageStore);
|
||||
|
||||
const languageInstruction = language !== 'en'
|
||||
? `You MUST respond in ${language} language. All output must be in ${language}. `
|
||||
// ? `You MUST respond in ${language} language. ALL output, including section headers, titles, and formatting, MUST be translated into ${language}. It is CRITICAL that you translate ALL headers, such as SUMMARY, IDEAS, QUOTES, TAKEAWAYS, MAIN POINTS, etc., into ${language}. Maintain markdown formatting in the response. Do not output any English headers.`
|
||||
: '';
|
||||
const languageInstruction =
|
||||
language !== "en"
|
||||
? `You MUST respond in ${language} language. All output must be in ${language}. `
|
||||
: // ? `You MUST respond in ${language} language. ALL output, including section headers, titles, and formatting, MUST be translated into ${language}. It is CRITICAL that you translate ALL headers, such as SUMMARY, IDEAS, QUOTES, TAKEAWAYS, MAIN POINTS, etc., into ${language}. Maintain markdown formatting in the response. Do not output any English headers.`
|
||||
"";
|
||||
|
||||
const finalSystemPrompt = languageInstruction + (systemPromptText ?? get(systemPrompt));
|
||||
const finalSystemPrompt =
|
||||
languageInstruction + (systemPromptText ?? get(systemPrompt));
|
||||
|
||||
const finalUserInput = language !== 'en'
|
||||
? `${userInput}\n\nIMPORTANT: Respond in ${language} language only.`
|
||||
: userInput;
|
||||
const finalUserInput =
|
||||
language !== "en"
|
||||
? `${userInput}\n\nIMPORTANT: Respond in ${language} language only.`
|
||||
: userInput;
|
||||
|
||||
return {
|
||||
userInput: finalUserInput,
|
||||
systemPrompt: finalSystemPrompt,
|
||||
model: config.model,
|
||||
patternName: get(selectedPatternName),
|
||||
strategyName: get(selectedStrategy), // Add selected strategy to prompt
|
||||
variables: get(patternVariables) // Add pattern variables
|
||||
};
|
||||
}
|
||||
|
||||
public async createChatRequest(userInput: string, systemPromptText?: string, isPattern: boolean = false): Promise<ChatRequest> {
|
||||
const prompt = this.createChatPrompt(userInput, systemPromptText);
|
||||
const config = get(chatConfig);
|
||||
const language = get(languageStore);
|
||||
|
||||
return {
|
||||
prompts: [prompt],
|
||||
messages: [],
|
||||
language: language, // Add language at the top level for backend compatibility
|
||||
...config
|
||||
};
|
||||
}
|
||||
|
||||
public async streamPattern(userInput: string, systemPromptText?: string): Promise<ReadableStream<StreamResponse>> {
|
||||
const request = await this.createChatRequest(userInput, systemPromptText, true);
|
||||
return this.fetchStream(request);
|
||||
}
|
||||
|
||||
public async streamChat(userInput: string, systemPromptText?: string): Promise<ReadableStream<StreamResponse>> {
|
||||
const request = await this.createChatRequest(userInput, systemPromptText);
|
||||
return this.fetchStream(request);
|
||||
}
|
||||
|
||||
public async processStream(
|
||||
stream: ReadableStream<StreamResponse>,
|
||||
onContent: (content: string, response?: StreamResponse) => void,
|
||||
onError: (error: Error) => void
|
||||
): Promise<void> {
|
||||
const reader = stream.getReader();
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
if (value.type === 'error') {
|
||||
throw new ChatError(value.content, 'STREAM_CONTENT_ERROR');
|
||||
}
|
||||
|
||||
if (value.type === 'content') {
|
||||
onContent(value.content, value);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
onError(error instanceof ChatError ? error : new ChatError('Stream processing error', 'STREAM_ERROR', error));
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
|
||||
}
|
||||
}
|
||||
return {
|
||||
userInput: finalUserInput,
|
||||
systemPrompt: finalSystemPrompt,
|
||||
model: config.model,
|
||||
patternName: get(selectedPatternName),
|
||||
strategyName: get(selectedStrategy), // Add selected strategy to prompt
|
||||
variables: get(patternVariables), // Add pattern variables
|
||||
};
|
||||
}
|
||||
|
||||
public async createChatRequest(
|
||||
userInput: string,
|
||||
systemPromptText?: string,
|
||||
isPattern: boolean = false,
|
||||
): Promise<ChatRequest> {
|
||||
const prompt = this.createChatPrompt(userInput, systemPromptText);
|
||||
const config = get(chatConfig);
|
||||
const language = get(languageStore);
|
||||
|
||||
return {
|
||||
prompts: [prompt],
|
||||
messages: [],
|
||||
language: language, // Add language at the top level for backend compatibility
|
||||
...config,
|
||||
};
|
||||
}
|
||||
|
||||
public async streamPattern(
|
||||
userInput: string,
|
||||
systemPromptText?: string,
|
||||
): Promise<ReadableStream<StreamResponse>> {
|
||||
const request = await this.createChatRequest(
|
||||
userInput,
|
||||
systemPromptText,
|
||||
true,
|
||||
);
|
||||
return this.fetchStream(request);
|
||||
}
|
||||
|
||||
public async streamChat(
|
||||
userInput: string,
|
||||
systemPromptText?: string,
|
||||
): Promise<ReadableStream<StreamResponse>> {
|
||||
const request = await this.createChatRequest(userInput, systemPromptText);
|
||||
return this.fetchStream(request);
|
||||
}
|
||||
|
||||
public async processStream(
|
||||
stream: ReadableStream<StreamResponse>,
|
||||
onContent: (content: string, response?: StreamResponse) => void,
|
||||
onError: (error: Error) => void,
|
||||
): Promise<void> {
|
||||
const reader = stream.getReader();
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
if (value.type === "error") {
|
||||
throw new ChatError(value.content, "STREAM_CONTENT_ERROR");
|
||||
}
|
||||
|
||||
if (value.type === "content") {
|
||||
onContent(value.content, value);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
onError(
|
||||
error instanceof ChatError
|
||||
? error
|
||||
: new ChatError("Stream processing error", "STREAM_ERROR", error),
|
||||
);
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,78 +1,74 @@
|
||||
import { createPipeline, transformers } from 'pdf-to-markdown-core/lib/src';
|
||||
import { PARSE_SCHEMA } from 'pdf-to-markdown-core/lib/src/PdfParser';
|
||||
import * as pdfjs from 'pdfjs-dist';
|
||||
import pdfConfig from './pdf-config';
|
||||
import { createPipeline, transformers } from "pdf-to-markdown-core/lib/src";
|
||||
import { PARSE_SCHEMA } from "pdf-to-markdown-core/lib/src/PdfParser";
|
||||
|
||||
// pdfjs-dist v5+ requires browser APIs at import time, so we use dynamic imports
|
||||
let pdfjs: typeof import("pdfjs-dist") | null = null;
|
||||
|
||||
export class PdfConversionService {
|
||||
constructor() {
|
||||
if (typeof window !== 'undefined') {
|
||||
console.log('PDF.js version:', pdfjs.version);
|
||||
// Initialize PDF.js configuration from the shared config
|
||||
pdfConfig.initialize();
|
||||
console.log('Worker configuration complete');
|
||||
}
|
||||
}
|
||||
private async ensureInitialized(): Promise<typeof import("pdfjs-dist")> {
|
||||
if (!pdfjs) {
|
||||
// Dynamic import to avoid SSR issues with pdfjs-dist v5+
|
||||
pdfjs = await import("pdfjs-dist");
|
||||
const pdfConfig = (await import("./pdf-config")).default;
|
||||
console.log("PDF.js version:", pdfjs.version);
|
||||
await pdfConfig.initialize();
|
||||
console.log("Worker configuration complete");
|
||||
}
|
||||
return pdfjs;
|
||||
}
|
||||
|
||||
async convertToMarkdown(file: File): Promise<string> {
|
||||
console.log('Starting PDF conversion:', {
|
||||
fileName: file.name,
|
||||
fileSize: file.size
|
||||
});
|
||||
async convertToMarkdown(file: File): Promise<string> {
|
||||
console.log("Starting PDF conversion:", {
|
||||
fileName: file.name,
|
||||
fileSize: file.size,
|
||||
});
|
||||
|
||||
const buffer = await file.arrayBuffer();
|
||||
console.log('Buffer created:', buffer.byteLength);
|
||||
const pdfjsLib = await this.ensureInitialized();
|
||||
|
||||
const pipeline = createPipeline(pdfjs, {
|
||||
transformConfig: {
|
||||
transformers
|
||||
}
|
||||
});
|
||||
console.log('Pipeline created');
|
||||
const buffer = await file.arrayBuffer();
|
||||
console.log("Buffer created:", buffer.byteLength);
|
||||
|
||||
const result = await pipeline.parse(
|
||||
buffer,
|
||||
(progress) => console.log('Processing:', {
|
||||
stage: progress.stages,
|
||||
details: progress.stageDetails,
|
||||
progress: progress.stageProgress
|
||||
})
|
||||
);
|
||||
console.log('Parse complete, validating result');
|
||||
const pipeline = createPipeline(pdfjsLib, {
|
||||
transformConfig: {
|
||||
transformers,
|
||||
},
|
||||
});
|
||||
console.log("Pipeline created");
|
||||
|
||||
const transformed = result.transform();
|
||||
console.log('Transform applied:', transformed);
|
||||
const result = await pipeline.parse(buffer, (progress) =>
|
||||
console.log("Processing:", {
|
||||
stage: progress.stages,
|
||||
details: progress.stageDetails,
|
||||
progress: progress.stageProgress,
|
||||
}),
|
||||
);
|
||||
console.log("Parse complete, validating result");
|
||||
|
||||
const markdown = transformed.convert({
|
||||
convert: (items) => {
|
||||
console.log('PDF Structure:', {
|
||||
itemCount: items.length,
|
||||
firstItem: items[0],
|
||||
schema: PARSE_SCHEMA // ['transform', 'width', 'height', 'str', 'fontName', 'dir']
|
||||
});
|
||||
|
||||
const text = items
|
||||
.map(item => item.value('str')) // Using 'str' instead of 'text' based on PARSE_SCHEMA
|
||||
.filter(Boolean)
|
||||
.join('\n');
|
||||
|
||||
console.log('Converted text:', {
|
||||
length: text.length,
|
||||
preview: text.substring(0, 100)
|
||||
});
|
||||
|
||||
return text;
|
||||
}
|
||||
});
|
||||
|
||||
const transformed = result.transform();
|
||||
console.log("Transform applied:", transformed);
|
||||
|
||||
return markdown;
|
||||
}
|
||||
const markdown = transformed.convert({
|
||||
convert: (items) => {
|
||||
console.log("PDF Structure:", {
|
||||
itemCount: items.length,
|
||||
firstItem: items[0],
|
||||
schema: PARSE_SCHEMA, // ['transform', 'width', 'height', 'str', 'fontName', 'dir']
|
||||
});
|
||||
|
||||
const text = items
|
||||
.map((item) => item.value("str")) // Using 'str' instead of 'text' based on PARSE_SCHEMA
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
|
||||
console.log("Converted text:", {
|
||||
length: text.length,
|
||||
preview: text.substring(0, 100),
|
||||
});
|
||||
|
||||
return text;
|
||||
},
|
||||
});
|
||||
|
||||
return markdown;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
import { browser } from '$app/environment';
|
||||
import { GlobalWorkerOptions } from 'pdfjs-dist';
|
||||
import { browser } from "$app/environment";
|
||||
|
||||
// Set up the worker source location - point to static file in public directory
|
||||
const workerSrc = '/pdf.worker.min.mjs';
|
||||
|
||||
// Configure the worker options only on the client side
|
||||
if (browser) {
|
||||
GlobalWorkerOptions.workerSrc = workerSrc;
|
||||
}
|
||||
|
||||
// Export the configuration
|
||||
// Export the configuration - accepts pdfjs module to avoid top-level import
|
||||
// This is necessary because pdfjs-dist v5+ uses browser APIs at import time
|
||||
export default {
|
||||
initialize: () => {
|
||||
if (browser) {
|
||||
console.log('PDF.js worker initialized at', workerSrc);
|
||||
}
|
||||
}
|
||||
};
|
||||
initialize: async () => {
|
||||
if (browser) {
|
||||
// Dynamic import to avoid SSR issues
|
||||
const pdfjs = await import("pdfjs-dist");
|
||||
const { GlobalWorkerOptions, version } = pdfjs;
|
||||
|
||||
// Use CDN-hosted worker to avoid bundling third-party minified code in the repo
|
||||
const workerSrc = `https://unpkg.com/pdfjs-dist@${version}/build/pdf.worker.min.mjs`;
|
||||
GlobalWorkerOptions.workerSrc = workerSrc;
|
||||
|
||||
console.log(`PDF.js worker v${version} initialized from CDN`);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
@@ -1,19 +1,24 @@
|
||||
import { writable, derived, get } from 'svelte/store';
|
||||
import type { ChatState, Message, StreamResponse } from '$lib/interfaces/chat-interface';
|
||||
import { ChatService, ChatError } from '$lib/services/ChatService';
|
||||
import { languageStore } from '$lib/store/language-store';
|
||||
import { selectedPatternName } from '$lib/store/pattern-store';
|
||||
import { derived, get, writable } from "svelte/store";
|
||||
import { browser } from "$app/environment";
|
||||
import type {
|
||||
ChatState,
|
||||
Message,
|
||||
StreamResponse,
|
||||
} from "$lib/interfaces/chat-interface";
|
||||
import { ChatError, ChatService } from "$lib/services/ChatService";
|
||||
import { languageStore } from "$lib/store/language-store";
|
||||
import { selectedPatternName } from "$lib/store/pattern-store";
|
||||
|
||||
// Initialize chat service
|
||||
const chatService = new ChatService();
|
||||
|
||||
// Local storage key for persisting messages
|
||||
const MESSAGES_STORAGE_KEY = 'chat_messages';
|
||||
const MESSAGES_STORAGE_KEY = "chat_messages";
|
||||
|
||||
// Load initial messages from local storage
|
||||
const initialMessages = typeof localStorage !== 'undefined'
|
||||
? JSON.parse(localStorage.getItem(MESSAGES_STORAGE_KEY) || '[]')
|
||||
: [];
|
||||
// Load initial messages from local storage (only in browser)
|
||||
const initialMessages = browser
|
||||
? JSON.parse(localStorage.getItem(MESSAGES_STORAGE_KEY) || "[]")
|
||||
: [];
|
||||
|
||||
// Separate stores for different concerns
|
||||
export const messageStore = writable<Message[]>(initialMessages);
|
||||
@@ -21,134 +26,144 @@ export const streamingStore = writable<boolean>(false);
|
||||
export const errorStore = writable<string | null>(null);
|
||||
export const currentSession = writable<string | null>(null);
|
||||
|
||||
// Subscribe to messageStore changes to persist messages
|
||||
if (typeof localStorage !== 'undefined') {
|
||||
messageStore.subscribe($messages => {
|
||||
localStorage.setItem(MESSAGES_STORAGE_KEY, JSON.stringify($messages));
|
||||
});
|
||||
// Subscribe to messageStore changes to persist messages (only in browser)
|
||||
if (browser) {
|
||||
messageStore.subscribe(($messages) => {
|
||||
localStorage.setItem(MESSAGES_STORAGE_KEY, JSON.stringify($messages));
|
||||
});
|
||||
}
|
||||
|
||||
// Derived store for chat state
|
||||
export const chatState = derived(
|
||||
[messageStore, streamingStore],
|
||||
([$messages, $streaming]) => ({
|
||||
messages: $messages,
|
||||
isStreaming: $streaming
|
||||
})
|
||||
[messageStore, streamingStore],
|
||||
([$messages, $streaming]) => ({
|
||||
messages: $messages,
|
||||
isStreaming: $streaming,
|
||||
}),
|
||||
);
|
||||
|
||||
// Error handling utility
|
||||
function handleError(error: Error | string) {
|
||||
const errorMessage = error instanceof ChatError
|
||||
? `${error.code}: ${error.message}`
|
||||
: error instanceof Error
|
||||
? error.message
|
||||
: error;
|
||||
const errorMessage =
|
||||
error instanceof ChatError
|
||||
? `${error.code}: ${error.message}`
|
||||
: error instanceof Error
|
||||
? error.message
|
||||
: error;
|
||||
|
||||
errorStore.set(errorMessage);
|
||||
streamingStore.set(false);
|
||||
return errorMessage;
|
||||
errorStore.set(errorMessage);
|
||||
streamingStore.set(false);
|
||||
return errorMessage;
|
||||
}
|
||||
|
||||
export const setSession = (sessionName: string | null) => {
|
||||
currentSession.set(sessionName);
|
||||
if (!sessionName) {
|
||||
clearMessages();
|
||||
}
|
||||
currentSession.set(sessionName);
|
||||
if (!sessionName) {
|
||||
clearMessages();
|
||||
}
|
||||
};
|
||||
|
||||
export const clearMessages = () => {
|
||||
messageStore.set([]);
|
||||
errorStore.set(null);
|
||||
if (typeof localStorage !== 'undefined') {
|
||||
localStorage.removeItem(MESSAGES_STORAGE_KEY);
|
||||
}
|
||||
messageStore.set([]);
|
||||
errorStore.set(null);
|
||||
if (typeof localStorage !== "undefined") {
|
||||
localStorage.removeItem(MESSAGES_STORAGE_KEY);
|
||||
}
|
||||
};
|
||||
|
||||
export const revertLastMessage = () => {
|
||||
messageStore.update(messages => messages.slice(0, -1));
|
||||
messageStore.update((messages) => messages.slice(0, -1));
|
||||
};
|
||||
|
||||
export async function sendMessage(
|
||||
content: string,
|
||||
systemPromptText?: string,
|
||||
isSystem: boolean = false,
|
||||
) {
|
||||
try {
|
||||
console.log("\n=== Message Processing Start ===");
|
||||
console.log("1. Initial state:", {
|
||||
isSystem,
|
||||
hasSystemPrompt: !!systemPromptText,
|
||||
currentLanguage: get(languageStore),
|
||||
pattern: get(selectedPatternName),
|
||||
});
|
||||
|
||||
export async function sendMessage(content: string, systemPromptText?: string, isSystem: boolean = false) {
|
||||
try {
|
||||
console.log('\n=== Message Processing Start ===');
|
||||
console.log('1. Initial state:', {
|
||||
isSystem,
|
||||
hasSystemPrompt: !!systemPromptText,
|
||||
currentLanguage: get(languageStore),
|
||||
pattern: get(selectedPatternName)
|
||||
});
|
||||
const $streaming = get(streamingStore);
|
||||
if ($streaming) {
|
||||
throw new ChatError(
|
||||
"Message submission blocked - already streaming",
|
||||
"STREAMING_BLOCKED",
|
||||
);
|
||||
}
|
||||
|
||||
const $streaming = get(streamingStore);
|
||||
if ($streaming) {
|
||||
throw new ChatError('Message submission blocked - already streaming', 'STREAMING_BLOCKED');
|
||||
}
|
||||
streamingStore.set(true);
|
||||
errorStore.set(null);
|
||||
|
||||
streamingStore.set(true);
|
||||
errorStore.set(null);
|
||||
// Add message
|
||||
messageStore.update((messages) => [
|
||||
...messages,
|
||||
{
|
||||
role: isSystem ? "system" : "user",
|
||||
content,
|
||||
},
|
||||
]);
|
||||
|
||||
// Add message
|
||||
messageStore.update(messages => [...messages, {
|
||||
role: isSystem ? 'system' : 'user',
|
||||
content
|
||||
}]);
|
||||
console.log("2. Message added:", {
|
||||
role: isSystem ? "system" : "user",
|
||||
language: get(languageStore),
|
||||
});
|
||||
|
||||
console.log('2. Message added:', {
|
||||
role: isSystem ? 'system' : 'user',
|
||||
language: get(languageStore)
|
||||
});
|
||||
if (!isSystem) {
|
||||
console.log("3. Preparing chat stream:", {
|
||||
language: get(languageStore),
|
||||
pattern: get(selectedPatternName),
|
||||
hasSystemPrompt: !!systemPromptText,
|
||||
});
|
||||
|
||||
if (!isSystem) {
|
||||
console.log('3. Preparing chat stream:', {
|
||||
language: get(languageStore),
|
||||
pattern: get(selectedPatternName),
|
||||
hasSystemPrompt: !!systemPromptText
|
||||
});
|
||||
const stream = await chatService.streamChat(content, systemPromptText);
|
||||
console.log("4. Stream created");
|
||||
|
||||
const stream = await chatService.streamChat(content, systemPromptText);
|
||||
console.log('4. Stream created');
|
||||
await chatService.processStream(
|
||||
stream,
|
||||
(content: string, response?: StreamResponse) => {
|
||||
messageStore.update((messages) => {
|
||||
const newMessages = [...messages];
|
||||
const lastMessage = newMessages[newMessages.length - 1];
|
||||
|
||||
await chatService.processStream(
|
||||
stream,
|
||||
(content: string, response?: StreamResponse) => {
|
||||
messageStore.update(messages => {
|
||||
const newMessages = [...messages];
|
||||
const lastMessage = newMessages[newMessages.length - 1];
|
||||
if (lastMessage?.role === "assistant") {
|
||||
lastMessage.content = content;
|
||||
lastMessage.format = response?.format;
|
||||
console.log("Message updated:", {
|
||||
role: "assistant",
|
||||
format: lastMessage.format,
|
||||
});
|
||||
} else {
|
||||
newMessages.push({
|
||||
role: "assistant",
|
||||
content,
|
||||
format: response?.format,
|
||||
});
|
||||
}
|
||||
|
||||
if (lastMessage?.role === 'assistant') {
|
||||
lastMessage.content = content;
|
||||
lastMessage.format = response?.format;
|
||||
console.log('Message updated:', {
|
||||
role: 'assistant',
|
||||
format: lastMessage.format
|
||||
});
|
||||
} else {
|
||||
newMessages.push({
|
||||
role: 'assistant',
|
||||
content,
|
||||
format: response?.format
|
||||
});
|
||||
}
|
||||
return newMessages;
|
||||
});
|
||||
},
|
||||
(error) => {
|
||||
handleError(error);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return newMessages;
|
||||
});
|
||||
},
|
||||
(error) => {
|
||||
handleError(error);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
streamingStore.set(false);
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
handleError(error);
|
||||
} else {
|
||||
handleError(String(error));
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
streamingStore.set(false);
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
handleError(error);
|
||||
} else {
|
||||
handleError(String(error));
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Re-export types for convenience
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user