1.1.46

feat(package.json): add @dqbd/tiktoken dependency
fix(tokenCount.ts): uncomment Tiktoken import and encoding, and return token length instead of an approximation

.yml

@@ -0,0 +1,2 @@
+- name: Close Stale Issues
+  uses: actions/stale@v4.1.1

esbuild.config.js

@@ -0,0 +1,14 @@
+import { build } from 'esbuild'
+import fs from 'fs'    
+
+await build({
+    entryPoints: ['./src/cli.ts'],
+    bundle: true,
+    platform: 'node',
+    format: 'cjs',
+    outfile: './out/cli.cjs',
+});
+
+const wasmFile = fs.readFileSync('./node_modules/@dqbd/tiktoken/lite/tiktoken_bg.wasm')
+
+fs.writeFileSync('./out/tiktoken_bg.wasm', wasmFile)

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "opencommit",
-  "version": "1.1.42",
+  "version": "1.1.46",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "opencommit",
-      "version": "1.1.42",
+      "version": "1.1.46",
       "license": "MIT",
       "dependencies": {
         "@clack/prompts": "^0.6.1",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencommit",
-  "version": "1.1.42",
+  "version": "1.1.46",
   "description": "GPT CLI to auto-generate impressive commits in 1 second. Killing lame commits with AI 🤯🔫",
   "keywords": [
     "git",
@@ -40,7 +40,7 @@
     "watch": "npm run -S build -- --sourcemap --watch",
     "start": "node ./out/cli.cjs",
     "dev": "ts-node ./src/cli.ts",
-    "build": "rimraf out && esbuild ./src/cli.ts --bundle --outfile=out/cli.cjs --format=cjs --platform=node",
+    "build": "rimraf out && node esbuild.config.js",
     "deploy": "npm run build && npm version patch && npm publish --tag latest",
     "lint": "eslint src --ext ts && tsc --noEmit",
     "format": "prettier --write src"
@@ -60,6 +60,7 @@
   },
   "dependencies": {
     "@clack/prompts": "^0.6.1",
+    "@dqbd/tiktoken": "^1.0.2",
     "axios": "^1.3.4",
     "chalk": "^5.2.0",
     "cleye": "^1.3.2",

src/utils/tokenCount.ts

@@ -1,14 +1,15 @@
-// import { Tiktoken } from '@dqbd/tiktoken/lite';
-// import cl100k_base from '@dqbd/tiktoken/encoders/cl100k_base.json' assert { type: 'json' };
+import { Tiktoken } from '@dqbd/tiktoken/lite';
+import cl100k_base from '@dqbd/tiktoken/encoders/cl100k_base.json' assert { type: 'json' };
 
 export function tokenCount(content: string): number {
-  //   const encoding = new Tiktoken(
-  //     cl100k_base.bpe_ranks,
-  //     cl100k_base.special_tokens,
-  //     cl100k_base.pat_str
-  //   );
-  //   const tokens = encoding.encode(content);
-  //   encoding.free();
+  const encoding = new Tiktoken(
+    cl100k_base.bpe_ranks,
+    cl100k_base.special_tokens,
+    cl100k_base.pat_str
+  );
+  const tokens = encoding.encode(content);
+  encoding.free();
+  return tokens.length;
 
-  return content.length / 2.7;
+  //return content.length / 2.7;
 }