From 566a9b1a520349979d0e623a7556be3977bb9054 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sat, 12 Apr 2025 04:13:42 -0400 Subject: [PATCH] reuse function Signed-off-by: Tiger Kaovilai --- src/engine/anthropic.ts | 8 +--- src/engine/azure.ts | 8 +--- src/engine/deepseek.ts | 8 +--- src/engine/flowise.ts | 8 +--- src/engine/gemini.ts | 8 +--- src/engine/mistral.ts | 8 +--- src/engine/mlx.ts | 9 +---- src/engine/ollama.ts | 8 +--- src/engine/openAi.ts | 8 +--- src/utils/removeContentTags.ts | 51 ++++++++++++++++++++++++++ test/unit/removeContentTags.test.ts | 57 +++++++++++++++++++++++++++++ 11 files changed, 126 insertions(+), 55 deletions(-) create mode 100644 src/utils/removeContentTags.ts create mode 100644 test/unit/removeContentTags.test.ts diff --git a/src/engine/anthropic.ts b/src/engine/anthropic.ts index 88480f6..7dbc593 100644 --- a/src/engine/anthropic.ts +++ b/src/engine/anthropic.ts @@ -8,6 +8,7 @@ import axios from 'axios'; import chalk from 'chalk'; import { OpenAI } from 'openai'; import { GenerateCommitMessageErrorEnum } from '../generateCommitMessageFromGitDiff'; +import { removeContentTags } from '../utils/removeContentTags'; import { tokenCount } from '../utils/tokenCount'; import { AiEngine, AiEngineConfig } from './Engine'; @@ -55,12 +56,7 @@ export class AnthropicEngine implements AiEngine { const message = data?.content[0].text; let content = message; - - if (content && content.includes('')) { - return content.replace(/[\s\S]*?<\/think>/g, '').trim(); - } - - return content; + return removeContentTags(content, 'think'); } catch (error) { const err = error as Error; outro(`${chalk.red('✖')} ${err?.message || err}`); diff --git a/src/engine/azure.ts b/src/engine/azure.ts index c2dc306..acbf0d3 100644 --- a/src/engine/azure.ts +++ b/src/engine/azure.ts @@ -7,6 +7,7 @@ import axios from 'axios'; import chalk from 'chalk'; import { OpenAI } from 'openai'; import { GenerateCommitMessageErrorEnum } from '../generateCommitMessageFromGitDiff'; +import { removeContentTags } from '../utils/removeContentTags'; import { tokenCount } from '../utils/tokenCount'; import { AiEngine, AiEngineConfig } from './Engine'; @@ -54,12 +55,7 @@ export class AzureEngine implements AiEngine { } let content = message?.content; - - if (content && content.includes('')) { - return content.replace(/[\s\S]*?<\/think>/g, '').trim(); - } - - return content; + return removeContentTags(content, 'think'); } catch (error) { outro(`${chalk.red('✖')} ${this.config.model}`); diff --git a/src/engine/deepseek.ts b/src/engine/deepseek.ts index 4ae2a68..61b2ceb 100644 --- a/src/engine/deepseek.ts +++ b/src/engine/deepseek.ts @@ -1,6 +1,7 @@ import axios from 'axios'; import { OpenAI } from 'openai'; import { GenerateCommitMessageErrorEnum } from '../generateCommitMessageFromGitDiff'; +import { removeContentTags } from '../utils/removeContentTags'; import { tokenCount } from '../utils/tokenCount'; import { OpenAiEngine, OpenAiConfig } from './openAi'; @@ -42,12 +43,7 @@ export class DeepseekEngine extends OpenAiEngine { const message = completion.choices[0].message; let content = message?.content; - - if (content && content.includes('')) { - return content.replace(/[\s\S]*?<\/think>/g, '').trim(); - } - - return content; + return removeContentTags(content, 'think'); } catch (error) { const err = error as Error; if ( diff --git a/src/engine/flowise.ts b/src/engine/flowise.ts index 7f49029..aecc277 100644 --- a/src/engine/flowise.ts +++ b/src/engine/flowise.ts @@ -1,5 +1,6 @@ import axios, { AxiosInstance } from 'axios'; import { OpenAI } from 'openai'; +import { removeContentTags } from '../utils/removeContentTags'; import { AiEngine, AiEngineConfig } from './Engine'; interface FlowiseAiConfig extends AiEngineConfig {} @@ -37,12 +38,7 @@ export class FlowiseEngine implements AiEngine { const response = await this.client.post('', payload); const message = response.data; let content = message?.text; - - if (content && content.includes('')) { - return content.replace(/[\s\S]*?<\/think>/g, '').trim(); - } - - return content; + return removeContentTags(content, 'think'); } catch (err: any) { const message = err.response?.data?.error ?? err.message; throw new Error('local model issues. details: ' + message); diff --git a/src/engine/gemini.ts b/src/engine/gemini.ts index d516451..d56cf07 100644 --- a/src/engine/gemini.ts +++ b/src/engine/gemini.ts @@ -7,6 +7,7 @@ import { } from '@google/generative-ai'; import axios from 'axios'; import { OpenAI } from 'openai'; +import { removeContentTags } from '../utils/removeContentTags'; import { AiEngine, AiEngineConfig } from './Engine'; interface GeminiConfig extends AiEngineConfig {} @@ -72,12 +73,7 @@ export class GeminiEngine implements AiEngine { }); const content = result.response.text(); - - if (content && content.includes('')) { - return content.replace(/[\s\S]*?<\/think>/g, '').trim(); - } - - return content; + return removeContentTags(content, 'think'); } catch (error) { const err = error as Error; if ( diff --git a/src/engine/mistral.ts b/src/engine/mistral.ts index a16a6dc..8a7e932 100644 --- a/src/engine/mistral.ts +++ b/src/engine/mistral.ts @@ -1,6 +1,7 @@ import axios from 'axios'; import { OpenAI } from 'openai'; import { GenerateCommitMessageErrorEnum } from '../generateCommitMessageFromGitDiff'; +import { removeContentTags } from '../utils/removeContentTags'; import { tokenCount } from '../utils/tokenCount'; import { AiEngine, AiEngineConfig } from './Engine'; @@ -58,12 +59,7 @@ export class MistralAiEngine implements AiEngine { throw Error('No completion choice available.') let content = message.content as string; - - if (content && content.includes('')) { - return content.replace(/[\s\S]*?<\/think>/g, '').trim(); - } - - return content; + return removeContentTags(content, 'think'); } catch (error) { const err = error as Error; if ( diff --git a/src/engine/mlx.ts b/src/engine/mlx.ts index 4abedc4..f83a19a 100644 --- a/src/engine/mlx.ts +++ b/src/engine/mlx.ts @@ -1,7 +1,7 @@ import axios, { AxiosInstance } from 'axios'; import { OpenAI } from 'openai'; +import { removeContentTags } from '../utils/removeContentTags'; import { AiEngine, AiEngineConfig } from './Engine'; -import { chown } from 'fs'; interface MLXConfig extends AiEngineConfig {} @@ -38,12 +38,7 @@ export class MLXEngine implements AiEngine { const choices = response.data.choices; const message = choices[0].message; let content = message?.content; - - if (content && content.includes('')) { - return content.replace(/[\s\S]*?<\/think>/g, '').trim(); - } - - return content; + return removeContentTags(content, 'think'); } catch (err: any) { const message = err.response?.data?.error ?? err.message; throw new Error(`MLX provider error: ${message}`); diff --git a/src/engine/ollama.ts b/src/engine/ollama.ts index af3b927..2d21d63 100644 --- a/src/engine/ollama.ts +++ b/src/engine/ollama.ts @@ -1,5 +1,6 @@ import axios, { AxiosInstance } from 'axios'; import { OpenAI } from 'openai'; +import { removeContentTags } from '../utils/removeContentTags'; import { AiEngine, AiEngineConfig } from './Engine'; interface OllamaConfig extends AiEngineConfig {} @@ -35,12 +36,7 @@ export class OllamaEngine implements AiEngine { const { message } = response.data; let content = message?.content; - - if (content && content.includes('')) { - return content.replace(/[\s\S]*?<\/think>/g, '').trim(); - } - - return content; + return removeContentTags(content, 'think'); } catch (err: any) { const message = err.response?.data?.error ?? err.message; throw new Error(`Ollama provider error: ${message}`); diff --git a/src/engine/openAi.ts b/src/engine/openAi.ts index 4c0c46c..4e1c6a9 100644 --- a/src/engine/openAi.ts +++ b/src/engine/openAi.ts @@ -1,6 +1,7 @@ import axios from 'axios'; import { OpenAI } from 'openai'; import { GenerateCommitMessageErrorEnum } from '../generateCommitMessageFromGitDiff'; +import { removeContentTags } from '../utils/removeContentTags'; import { tokenCount } from '../utils/tokenCount'; import { AiEngine, AiEngineConfig } from './Engine'; @@ -46,12 +47,7 @@ export class OpenAiEngine implements AiEngine { const message = completion.choices[0].message; let content = message?.content; - - if (content && content.includes('')) { - return content.replace(/[\s\S]*?<\/think>/g, '').trim(); - } - - return content; + return removeContentTags(content, 'think'); } catch (error) { const err = error as Error; if ( diff --git a/src/utils/removeContentTags.ts b/src/utils/removeContentTags.ts new file mode 100644 index 0000000..d478434 --- /dev/null +++ b/src/utils/removeContentTags.ts @@ -0,0 +1,51 @@ +/** + * Removes content wrapped in specified tags from a string + * @param content The content string to process + * @param tag The tag name without angle brackets (e.g., 'think' for '') + * @returns The content with the specified tags and their contents removed, and trimmed + */ +export function removeContentTags(content: T, tag: string): T { + if (!content || typeof content !== 'string') { + return content; + } + + // Dynamic implementation for other cases + const openTag = `<${tag}>`; + const closeTag = ``; + + // Parse the content and remove tags + let result = ''; + let skipUntil: number | null = null; + let depth = 0; + + for (let i = 0; i < content.length; i++) { + // Check for opening tag + if (content.substring(i, i + openTag.length) === openTag) { + depth++; + if (depth === 1) { + skipUntil = content.indexOf(closeTag, i + openTag.length); + i = i + openTag.length - 1; // Skip the opening tag + continue; + } + } + // Check for closing tag + else if (content.substring(i, i + closeTag.length) === closeTag && depth > 0) { + depth--; + if (depth === 0) { + i = i + closeTag.length - 1; // Skip the closing tag + skipUntil = null; + continue; + } + } + + // Only add character if not inside a tag + if (skipUntil === null) { + result += content[i]; + } + } + + // Normalize spaces (replace multiple spaces with a single space) + result = result.replace(/\s+/g, ' ').trim(); + + return result as unknown as T; +} diff --git a/test/unit/removeContentTags.test.ts b/test/unit/removeContentTags.test.ts new file mode 100644 index 0000000..96f28fa --- /dev/null +++ b/test/unit/removeContentTags.test.ts @@ -0,0 +1,57 @@ +import { removeContentTags } from '../../src/utils/removeContentTags'; + +describe('removeContentTags', () => { + it('should remove content wrapped in specified tags', () => { + const content = 'This is something to hide visible content'; + const result = removeContentTags(content, 'think'); + expect(result).toBe('This is visible content'); + }); + + it('should handle multiple tag occurrences', () => { + const content = 'hidden visible also hidden text'; + const result = removeContentTags(content, 'think'); + expect(result).toBe('visible text'); + }); + + it('should handle multiline content within tags', () => { + const content = 'Start hidden\nover multiple\nlines End'; + const result = removeContentTags(content, 'think'); + expect(result).toBe('Start End'); + }); + + it('should return content as is when tag is not found', () => { + const content = 'Content without any tags'; + const result = removeContentTags(content, 'think'); + expect(result).toBe('Content without any tags'); + }); + + it('should work with different tag names', () => { + const content = 'This is something to hide visible content'; + const result = removeContentTags(content, 'custom'); + expect(result).toBe('This is visible content'); + }); + + it('should handle null content', () => { + const content = null; + const result = removeContentTags(content, 'think'); + expect(result).toBe(null); + }); + + it('should handle undefined content', () => { + const content = undefined; + const result = removeContentTags(content, 'think'); + expect(result).toBe(undefined); + }); + + it('should trim the result', () => { + const content = ' hidden visible '; + const result = removeContentTags(content, 'think'); + expect(result).toBe('visible'); + }); + + it('should handle nested tags correctly', () => { + const content = 'Outside Inside Nested End'; + const result = removeContentTags(content, 'think'); + expect(result).toBe('Outside End'); + }); +});