github/opencommit
1
1
Fork 0
mirror of https://github.com/di-sukharev/opencommit.git synced 2026-04-20 03:02:51 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev'

Browse Source
This commit is contained in:
di-sukharev
2023-04-05 12:44:30 +08:00
parent d5f53fec5a 9f65c450e3
commit 36f282d8a5
3 changed files with 27 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
esbuild.config.js Normal file
View File

@@ -0,0 +1,14 @@
import { build } from 'esbuild'
import fs from 'fs'
await build({
entryPoints: ['./src/cli.ts'],
bundle: true,
platform: 'node',
format: 'cjs',
outfile: './out/cli.cjs',
});
const wasmFile = fs.readFileSync('./node_modules/@dqbd/tiktoken/lite/tiktoken_bg.wasm')
fs.writeFileSync('./out/tiktoken_bg.wasm', wasmFile)

3
package.json
View File

@@ -40,7 +40,7 @@
"watch": "npm run -S build -- --sourcemap --watch",
"start": "node ./out/cli.cjs",
"dev": "ts-node ./src/cli.ts",
"build": "rimraf out && esbuild ./src/cli.ts --bundle --outfile=out/cli.cjs --format=cjs --platform=node",
"build": "rimraf out && node esbuild.config.js",
"deploy": "npm run build && npm version patch && npm publish --tag latest",
"lint": "eslint src --ext ts && tsc --noEmit",
"format": "prettier --write src"
@@ -60,6 +60,7 @@
},
"dependencies": {
"@clack/prompts": "^0.6.1",
"@dqbd/tiktoken": "^1.0.2",
"axios": "^1.3.4",
"chalk": "^5.2.0",
"cleye": "^1.3.2",

21
src/utils/tokenCount.ts
View File

@@ -1,14 +1,15 @@
// import { Tiktoken } from '@dqbd/tiktoken/lite';
// import cl100k_base from '@dqbd/tiktoken/encoders/cl100k_base.json' assert { type: 'json' };
import { Tiktoken } from '@dqbd/tiktoken/lite';
import cl100k_base from '@dqbd/tiktoken/encoders/cl100k_base.json' assert { type: 'json' };
export function tokenCount(content: string): number {
// const encoding = new Tiktoken(
// cl100k_base.bpe_ranks,
// cl100k_base.special_tokens,
// cl100k_base.pat_str
// );
// const tokens = encoding.encode(content);
// encoding.free();
const encoding = new Tiktoken(
cl100k_base.bpe_ranks,
cl100k_base.special_tokens,
cl100k_base.pat_str
);
const tokens = encoding.encode(content);
encoding.free();
return tokens.length;
return content.length / 2.7;
//return content.length / 2.7;
}