github/opencommit
1
1
Fork 0
mirror of https://github.com/di-sukharev/opencommit.git synced 2026-01-30 07:58:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactor(tokenCount.ts): format code with consistent indentation and quotes

Browse Source 
The code has been reformatted to use consistent indentation and quotes. This improves the readability and maintainability of the code. No functionality has been changed.
This commit is contained in:
di-sukharev
2023-03-29 11:25:48 +08:00
parent 0e1ad33179
commit 853662acc4
1 changed files with 10 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/utils/tokenCount.ts
View File

@@ -1,14 +1,14 @@
import { Tiktoken } from "@dqbd/tiktoken/lite"
import cl100k_base from "@dqbd/tiktoken/encoders/cl100k_base.json" assert{type: "json"}
import { Tiktoken } from '@dqbd/tiktoken/lite';
import cl100k_base from '@dqbd/tiktoken/encoders/cl100k_base.json' assert { type: 'json' };
export function tokenCount(content: string): number {
const encoding = new Tiktoken(
cl100k_base.bpe_ranks,
cl100k_base.special_tokens,
cl100k_base.pat_str
);
const tokens = encoding.encode(content);
encoding.free();
const encoding = new Tiktoken(
cl100k_base.bpe_ranks,
cl100k_base.special_tokens,
cl100k_base.pat_str
);
const tokens = encoding.encode(content);
encoding.free();
return tokens.length;
return tokens.length;
}