mirror of
https://github.com/Significant-Gravitas/AutoGPT.git
synced 2026-04-08 03:00:28 -04:00
Adds necessary spaces
Introduces spaces between code blocks.
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import tiktoken
|
||||
from typing import List, Dict
|
||||
|
||||
|
||||
def count_message_tokens(messages : List[Dict[str, str]], model : str = "gpt-3.5-turbo-0301") -> int:
|
||||
"""
|
||||
Returns the number of tokens used by a list of messages.
|
||||
@@ -17,6 +18,7 @@ def count_message_tokens(messages : List[Dict[str, str]], model : str = "gpt-3.5
|
||||
except KeyError:
|
||||
print("Warning: model not found. Using cl100k_base encoding.")
|
||||
encoding = tiktoken.get_encoding("cl100k_base")
|
||||
|
||||
if model == "gpt-3.5-turbo":
|
||||
# !Node: gpt-3.5-turbo may change over time. Returning num tokens assuming gpt-3.5-turbo-0301.")
|
||||
return count_message_tokens(messages, model="gpt-3.5-turbo-0301")
|
||||
@@ -32,15 +34,19 @@ def count_message_tokens(messages : List[Dict[str, str]], model : str = "gpt-3.5
|
||||
else:
|
||||
raise NotImplementedError(f"""num_tokens_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.""")
|
||||
num_tokens = 0
|
||||
|
||||
for message in messages:
|
||||
num_tokens += tokens_per_message
|
||||
|
||||
for key, value in message.items():
|
||||
num_tokens += len(encoding.encode(value))
|
||||
|
||||
if key == "name":
|
||||
num_tokens += tokens_per_name
|
||||
num_tokens += 3 # every reply is primed with <|start|>assistant<|message|>
|
||||
return num_tokens
|
||||
|
||||
|
||||
def count_string_tokens(string: str, model_name: str) -> int:
|
||||
"""
|
||||
Returns the number of tokens in a text string.
|
||||
|
||||
Reference in New Issue
Block a user