diff --git a/src/utils/tokenCount.ts b/src/utils/tokenCount.ts index 84e4f23..676445d 100644 --- a/src/utils/tokenCount.ts +++ b/src/utils/tokenCount.ts @@ -1,14 +1,14 @@ -import { Tiktoken } from "@dqbd/tiktoken/lite" -import cl100k_base from "@dqbd/tiktoken/encoders/cl100k_base.json" assert{type: "json"} +import { Tiktoken } from '@dqbd/tiktoken/lite'; +import cl100k_base from '@dqbd/tiktoken/encoders/cl100k_base.json' assert { type: 'json' }; export function tokenCount(content: string): number { - const encoding = new Tiktoken( - cl100k_base.bpe_ranks, - cl100k_base.special_tokens, - cl100k_base.pat_str - ); - const tokens = encoding.encode(content); - encoding.free(); + const encoding = new Tiktoken( + cl100k_base.bpe_ranks, + cl100k_base.special_tokens, + cl100k_base.pat_str + ); + const tokens = encoding.encode(content); + encoding.free(); - return tokens.length; + return tokens.length; }