mirror of
https://github.com/di-sukharev/opencommit.git
synced 2026-04-20 03:02:51 -04:00
1b70de1d20
chore(githook.ts): remove unused import refactor(tokenCount.ts): simplify token count calculation The console output now correctly capitalizes OpenCommit for consistency with the project name. The unused import of fileURLToPath is removed. The tokenCount function is refactored to simplify the calculation of the token count. The Tiktoken library is no longer used, and the token count is now calculated based on the length of the content divided by an average token length of 2.7 characters.
15 lines
453 B
TypeScript
15 lines
453 B
TypeScript
// import { Tiktoken } from '@dqbd/tiktoken/lite';
|
|
// import cl100k_base from '@dqbd/tiktoken/encoders/cl100k_base.json' assert { type: 'json' };
|
|
|
|
export function tokenCount(content: string): number {
|
|
// const encoding = new Tiktoken(
|
|
// cl100k_base.bpe_ranks,
|
|
// cl100k_base.special_tokens,
|
|
// cl100k_base.pat_str
|
|
// );
|
|
// const tokens = encoding.encode(content);
|
|
// encoding.free();
|
|
|
|
return content.length / 2.7;
|
|
}
|