Adds necessary spaces

Introduces spaces between code blocks.
This commit is contained in:
Andres Caicedo
2023-04-04 11:36:50 +02:00
parent 621f18eba7
commit 62615cacc9
4 changed files with 12 additions and 2 deletions

View File

@@ -1,6 +1,7 @@
import tiktoken
from typing import List, Dict
def count_message_tokens(messages : List[Dict[str, str]], model : str = "gpt-3.5-turbo-0301") -> int:
"""
Returns the number of tokens used by a list of messages.
@@ -17,6 +18,7 @@ def count_message_tokens(messages : List[Dict[str, str]], model : str = "gpt-3.5
except KeyError:
print("Warning: model not found. Using cl100k_base encoding.")
encoding = tiktoken.get_encoding("cl100k_base")
if model == "gpt-3.5-turbo":
# !Node: gpt-3.5-turbo may change over time. Returning num tokens assuming gpt-3.5-turbo-0301.")
return count_message_tokens(messages, model="gpt-3.5-turbo-0301")
@@ -32,15 +34,19 @@ def count_message_tokens(messages : List[Dict[str, str]], model : str = "gpt-3.5
else:
raise NotImplementedError(f"""num_tokens_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.""")
num_tokens = 0
for message in messages:
num_tokens += tokens_per_message
for key, value in message.items():
num_tokens += len(encoding.encode(value))
if key == "name":
num_tokens += tokens_per_name
num_tokens += 3 # every reply is primed with <|start|>assistant<|message|>
return num_tokens
def count_string_tokens(string: str, model_name: str) -> int:
"""
Returns the number of tokens in a text string.