Merge branch 'master' into dev

package-lock.json

@@ -1,15 +1,16 @@
 {
   "name": "opencommit",
-  "version": "1.1.22",
+  "version": "1.1.24",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "opencommit",
-      "version": "1.1.22",
+      "version": "1.1.24",
       "license": "MIT",
       "dependencies": {
         "@clack/prompts": "^0.6.1",
+        "@dqbd/tiktoken": "^1.0.2",
         "axios": "^1.3.4",
         "chalk": "^5.2.0",
         "cleye": "^1.3.2",
@@ -83,6 +84,11 @@
         "node": ">=12"
       }
     },
+    "node_modules/@dqbd/tiktoken": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@dqbd/tiktoken/-/tiktoken-1.0.2.tgz",
+      "integrity": "sha512-AjGTBRWsMoVmVeN55NLyupyM8TNamOUBl6tj5t/leLDVup3CFGO9tVagNL1jf3GyZLkWZSTmYVbPQ/M2LEcNzw=="
+    },
     "node_modules/@esbuild/android-arm": {
       "version": "0.15.18",
       "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.15.18.tgz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencommit",
-  "version": "1.1.22",
+  "version": "1.1.24",
   "description": "GPT CLI to auto-generate impressive commits in 1 second. Killing lame commits with AI 🤯🔫",
   "keywords": [
     "git",
@@ -60,6 +60,7 @@
   },
   "dependencies": {
     "@clack/prompts": "^0.6.1",
+    "@dqbd/tiktoken": "^1.0.2",
     "axios": "^1.3.4",
     "chalk": "^5.2.0",
     "cleye": "^1.3.2",

src/generateCommitMessageFromGitDiff.ts

@@ -6,6 +6,7 @@ import { api } from './api';
 import { getConfig } from './commands/config';
 import { mergeStrings } from './utils/mergeStrings';
 import { i18n, I18nLocals } from './i18n';
+import { tokenCount } from './utils/tokenCount';
 
 const config = getConfig();
 const translation = i18n[(config?.language as I18nLocals) || 'en'];
@@ -22,7 +23,10 @@ const INIT_MESSAGES_PROMPT: Array<ChatCompletionRequestMessage> = [
       config?.description
         ? 'Add a short description of why the commit is done after the commit message. Don\'t start it with "This commit", just describe the changes'
         : "Don't add any descriptions to the commit, only commit message"
-    }. Use the present tense. Use ${translation.localLanguage} to answer.`
+    }.
+    Use the present tense.
+    Lines must not be longer 74 characters.
+    Use ${translation.localLanguage} to answer.`
   },
   {
     role: ChatCompletionRequestMessageRoleEnum.User,
@@ -81,8 +85,8 @@ interface GenerateCommitMessageError {
 }
 
 const INIT_MESSAGES_PROMPT_LENGTH = INIT_MESSAGES_PROMPT.map(
-  (msg) => msg.content
-).join('').length;
+  (msg) => tokenCount(msg.content) + 4
+).reduce((a, b) => a + b, 0);
 
 const MAX_REQ_TOKENS = 3900 - INIT_MESSAGES_PROMPT_LENGTH;
 
@@ -90,7 +94,7 @@ export const generateCommitMessageWithChatCompletion = async (
   diff: string
 ): Promise<string | GenerateCommitMessageError> => {
   try {
-    if (diff.length >= MAX_REQ_TOKENS) {
+    if (tokenCount(diff) >= MAX_REQ_TOKENS) {
       const commitMessagePromises = getCommitMsgsPromisesFromFileDiffs(diff);
 
       const commitMessages = await Promise.all(commitMessagePromises);
@@ -145,7 +149,7 @@ function getCommitMsgsPromisesFromFileDiffs(diff: string) {
   const commitMessagePromises = [];
 
   for (const fileDiff of mergedFilesDiffs) {
-    if (fileDiff.length >= MAX_REQ_TOKENS) {
+    if (tokenCount(fileDiff) >= MAX_REQ_TOKENS) {
       // if file-diff is bigger than gpt context — split fileDiff into lineDiff
       const messagesPromises = getMessagesPromisesByLines(fileDiff, separator);
 

src/i18n/index.ts

@@ -9,6 +9,7 @@ import ja from '../i18n/ja.json' assert { type: 'json' };
 import pt_br from '../i18n/pt_br.json' assert { type: 'json' };
 import vi_VN from '../i18n/vi_VN.json' assert { type: 'json' };
 import es_ES from '../i18n/es_ES.json' assert { type: 'json' };
+import sv from '../i18n/sv.json' assert { type: 'json' };
 
 export enum I18nLocals {
   'en' = 'en',
@@ -20,7 +21,8 @@ export enum I18nLocals {
   'it' = 'it',
   'ko' = 'ko',
   'pt_br' = 'pt_br',
-  'es_ES' = 'es_ES'
+  'es_ES' = 'es_ES',
+  'sv' = 'sv',
 };
 
 export const i18n = {
@@ -34,7 +36,8 @@ export const i18n = {
   ko,
   pt_br,
   vi_VN,
-  es_ES
+  es_ES,
+  sv
 };
 
 export const I18N_CONFIG_ALIAS: { [key: string]: string[] } = {
@@ -49,6 +52,7 @@ export const I18N_CONFIG_ALIAS: { [key: string]: string[] } = {
   vi_VN: ['vi_VN', 'Vietnamese', 'tiếng Việt'],
   en: ['en', 'English', 'english'],
   es_ES: ['es_ES', 'Spanish', 'español'],
+  sv: ['sv', 'Swedish', 'Svenska'],
 };
 
 export function getI18nLocal(value: string): string | boolean {

src/i18n/sv.json

@@ -0,0 +1,6 @@
+{
+"localLanguage": "svenska",
+"commitFix": "fixa(server.ts): ändra variabelnamnet för port från små bokstäver till stora bokstäver PORT",
+"commitFeat": "nyhet(server.ts): lägg till stöd för process.env.PORT miljövariabel",
+"commitDescription": "Variabeln som innehåller portnumret heter nu PORT vilket förbättrar konsekvensen med namngivningskonventionerna eftersom PORT är en konstant. Stöd för en miljövariabel gör att applikationen kan vara mer flexibel då den nu kan köras på vilken port som helst som specificeras via miljövariabeln process.env.PORT."
+}

src/utils/mergeStrings.ts

@@ -1,8 +1,9 @@
+import { tokenCount } from './tokenCount'
 export function mergeStrings(arr: string[], maxStringLength: number): string[] {
   const mergedArr: string[] = [];
   let currentItem: string = arr[0];
   for (const item of arr.slice(1)) {
-    if (currentItem.length + item.length <= maxStringLength) {
+    if (tokenCount(currentItem + item) <= maxStringLength) {
       currentItem += item;
     } else {
       mergedArr.push(currentItem);

src/utils/tokenCount.ts

@@ -0,0 +1,14 @@
+import { Tiktoken } from "@dqbd/tiktoken/lite"
+import cl100k_base from "@dqbd/tiktoken/encoders/cl100k_base.json" assert{type: "json"}
+
+export function tokenCount(content: string): number {
+    const encoding = new Tiktoken(
+        cl100k_base.bpe_ranks,
+        cl100k_base.special_tokens,
+        cl100k_base.pat_str
+    );
+    const tokens = encoding.encode(content);
+    encoding.free();
+
+    return tokens.length;
+}