github/tinygrad
1
1
Fork 0
mirror of https://github.com/tinygrad/tinygrad.git synced 2026-01-09 06:58:11 -05:00
Code Issues Packages Projects Releases Wiki Activity

bring tinychat more inline with tinyos' version (#5358)

Browse Source
This commit is contained in:
wozeparrot
2024-07-10 20:13:52 +00:00
committed by GitHub
parent 84839f6c58
commit fa873df9c1
6 changed files with 40 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
examples/llama3.py
View File

@@ -12,6 +12,7 @@ class Tokenizer:
pat_str = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
def __init__(self, model_path: str):
mergeable_ranks = load_tiktoken_bpe(model_path)
self.num_base_tokens = len(mergeable_ranks)
special_tokens = [
"<|begin_of_text|>",
"<|end_of_text|>",
@@ -36,7 +37,7 @@ class Tokenizer:
@property
def stop_tokens(self): return {self.special_tokens["<|end_of_text|>"], self.special_tokens["<|eot_id|>"]}
def decode(self, toks): return self.model.decode(toks)
def decode(self, toks): return self.model.decode([t for t in toks if t < self.num_base_tokens])
def encode(self, text, allow_special=False):
return self.model.encode(text, allowed_special="all" if allow_special else set(), disallowed_special=set())
@@ -181,7 +182,7 @@ def build_transformer(model_path: Path, model_size="8B", quantize=None, device=N
TEMPERATURE = 0.85
TOP_K = 25
TOP_P = 0.9
ALPHA_F = 1.1
ALPHA_F = 0.1
ALPHA_P = 0.0
last_seen_toks = []
@@ -320,7 +321,7 @@ if __name__ == "__main__":
toks = [tokenizer.bos_id]
for message in rjson["messages"]:
toks += encode_message(message["role"], message["content"])
if message["role"] == "user":
if len(rjson["messages"]) > 0 and message["role"] == "user":
toks += encode_role("assistant")
return json.dumps(toks)

25
examples/tinychat/favicon.svg Normal file
View File

@@ -0,0 +1,25 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="-10 -10 150 70" shape-rendering="crispEdges">
<g id="logo">
<!-- t -->
<polygon points="10,40 10,20 0,20 0,10 10,10 10,0 20,0 20,10 30,10 30,20 20,20 20,30 30,30 30,40" />
<!-- i -->
<polygon points="40,40 40,20 50,20 50,40" />
<polygon points="40,10 40,0 50,0 50,10" />
<!-- n -->
<polygon points="60,40 60,10 80,10 80,40 90,40 90,20 70,20 70,40" />
<!-- y -->
<polygon points="100,50 100,40 130,40 130,10 120,10 120,20 110,20 110,10 100,10 100,30 120,30 120,50" />
</g>
<style>
@media (prefers-color-scheme: dark) {
#logo {
fill: #fff;
}
}
@media (prefers-color-scheme: light) {
#logo {
fill: #000;
}
}
</style>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 750 B

3
examples/tinychat/index.css
View File

@@ -47,7 +47,8 @@ main {
.title {
font-size: 3rem;
margin: 3rem 0;
margin: 1rem 0;
margin-top: 3rem;
}
.histories-container-container {

1
examples/tinychat/index.html
View File

@@ -3,6 +3,7 @@
<head>
<title>tinychat</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="icon" href="favicon.svg" type="image/svg+xml">
<script defer src="https://cdn.jsdelivr.net/npm/@alpine-collective/toolkit@1.0.2/dist/cdn.min.js"></script>
<script defer src="https://cdn.jsdelivr.net/npm/@alpinejs/intersect@3.x.x/dist/cdn.min.js"></script>

10
examples/tinychat/index.js
View File

@@ -74,7 +74,7 @@ document.addEventListener("alpine:init", () => {
start_time = Date.now();
this.time_till_first = start_time - prefill_start;
} else {
const diff = Date.now() - start_time
const diff = Date.now() - start_time;
if (diff > 0) {
this.tokens_per_second = tokens / (diff / 1000);
}
@@ -108,10 +108,10 @@ document.addEventListener("alpine:init", () => {
updateTotalTokens(messages) {
fetch(`${this.endpoint}/chat/token/encode`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ messages })
}).then(response => response.json()).then(data => {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ messages }),
}).then((response) => response.json()).then((data) => {
this.total_tokens = data.length;
}).catch(console.error);
},

3
extra/models/llama.py
View File

@@ -109,6 +109,9 @@ def sample(logits: Tensor, temp: float, k: int, p: float, af: float, ap: float):
setattr(sample, "alpha_counter", Tensor.zeros_like(logits, dtype=dtypes.int32).contiguous())
logits = logits - (sample.alpha_counter * af + (sample.alpha_counter > 0) * ap)
# replace NaNs with -inf
logits = (logits != logits).where(-float("inf"), logits)
# softmax
t = (logits / temp).softmax()