mirror of
https://github.com/tinygrad/tinygrad.git
synced 2026-01-08 22:48:25 -05:00
more tinychat fixes (#4971)
This commit is contained in:
wozeparrot
@@ -140,7 +140,7 @@ main {
|
||||
gap: 1rem;
|
||||
align-items: center;
|
||||
padding-top: 1rem;
|
||||
padding-bottom: 9rem;
|
||||
padding-bottom: 11rem;
|
||||
}
|
||||
|
||||
.message {
|
||||
@@ -153,7 +153,7 @@ main {
|
||||
padding: 0.5rem 1rem;
|
||||
border-radius: 10px;
|
||||
}
|
||||
.message-role-ai {
|
||||
.message-role-assistant {
|
||||
border-bottom: 2px solid var(--primary-color);
|
||||
border-left: 2px solid var(--primary-color);
|
||||
box-shadow: -10px 10px 20px 2px var(--primary-color-transparent);
|
||||
@@ -204,11 +204,31 @@ main {
|
||||
|
||||
width: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
z-index: 999;
|
||||
}
|
||||
|
||||
.input-performance {
|
||||
margin-top: 4rem;
|
||||
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.input-performance-point {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
place-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
.input-performance-point > p {
|
||||
height: 1rem;
|
||||
line-height: normal;
|
||||
}
|
||||
|
||||
.input {
|
||||
width: 90%;
|
||||
min-height: 3rem;
|
||||
@@ -221,7 +241,6 @@ main {
|
||||
|
||||
align-items: flex-end;
|
||||
margin-bottom: 2rem;
|
||||
margin-top: 4rem;
|
||||
}
|
||||
|
||||
.input-form {
|
||||
@@ -267,3 +286,7 @@ p {
|
||||
font-weight: 400;
|
||||
font-style: normal;
|
||||
}
|
||||
|
||||
.monospace {
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
@@ -7,16 +7,14 @@
|
||||
<script defer src="https://cdn.jsdelivr.net/npm/@alpine-collective/toolkit@1.0.2/dist/cdn.min.js"></script>
|
||||
<script defer src="https://cdn.jsdelivr.net/npm/@alpinejs/intersect@3.x.x/dist/cdn.min.js"></script>
|
||||
<script defer src="https://cdn.jsdelivr.net/npm/@alpinejs/focus@3.x.x/dist/cdn.min.js"></script>
|
||||
<script defer
|
||||
src="https://cdn.jsdelivr.net/npm/@marcreichel/alpine-auto-animate@latest/dist/alpine-auto-animate.min.js"></script>
|
||||
<script defer
|
||||
src="https://cdn.jsdelivr.net/npm/@marcreichel/alpine-autosize@latest/dist/alpine-autosize.min.js"></script>
|
||||
<script defer src="https://unpkg.com/alpinejs-swipe@1.0.2/dist/cjs.js"></script>
|
||||
<script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
|
||||
<script defer src="https://unpkg.com/@marcreichel/alpine-autosize@1.3.x/dist/alpine-autosize.min.js"></script>
|
||||
<script defer src="https://unpkg.com/alpinejs@3.x.x/dist/cdn.min.js"></script>
|
||||
|
||||
<script src="https://unpkg.com/dompurify@3.1.5/dist/purify.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/marked/lib/marked.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/marked-highlight/lib/index.umd.js"></script>
|
||||
<script src="https://unpkg.com/marked@13.0.0/marked.min.js"></script>
|
||||
<script src="https://unpkg.com/marked-highlight@2.1.2/lib/index.umd.js"></script>
|
||||
<script src="https://unpkg.com/@highlightjs/cdn-assets@11.9.0/highlight.min.js"></script>
|
||||
|
||||
<script src="index.js"></script>
|
||||
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
@@ -28,6 +26,7 @@
|
||||
integrity="sha512-SnH5WK+bZxgPHs44uWIX+LLJAJ9/2PkPKZ5QiAj6Ta86w+fsb2TkcmfRyVX3pBnMFcV7oQPJkl9QevSCWr3W6A=="
|
||||
crossorigin="anonymous" referrerpolicy="no-referrer" />
|
||||
<link rel="stylesheet" href="https://unpkg.com/@highlightjs/cdn-assets@11.9.0/styles/vs2015.min.css">
|
||||
|
||||
<link rel="stylesheet" href="index.css">
|
||||
<link rel="stylesheet" href="common.css">
|
||||
</head>
|
||||
@@ -42,6 +41,9 @@
|
||||
if (home === 2) {
|
||||
home = -1;
|
||||
cstate = { time: null, messages: [] };
|
||||
time_till_first = 0;
|
||||
tokens_per_second = 0;
|
||||
total_tokens = 0;
|
||||
}
|
||||
">
|
||||
<h1 class="title megrim-regular">tinychat</h1>
|
||||
@@ -55,6 +57,7 @@
|
||||
<template x-for="_state in histories.toSorted((a, b) => b.time - a.time)">
|
||||
<div x-data="{ otx: 0, trigger: 75 }" class="history" @click="
|
||||
cstate = _state;
|
||||
updateTotalTokens(cstate.messages);
|
||||
home = 1;
|
||||
// ensure that going back in history will go back to home
|
||||
window.history.pushState({}, '', '/');
|
||||
@@ -88,7 +91,12 @@
|
||||
value.messages.forEach(({ role, content }) => {
|
||||
const div = document.createElement('div');
|
||||
div.className = `message message-role-${role}`;
|
||||
div.innerHTML = DOMPurify.sanitize(marked.parse(content));
|
||||
try {
|
||||
div.innerHTML = DOMPurify.sanitize(marked.parse(content));
|
||||
} catch (e) {
|
||||
console.log(content);
|
||||
console.error(e);
|
||||
}
|
||||
|
||||
// add a clipboard button to all code blocks
|
||||
const codeBlocks = div.querySelectorAll('.hljs');
|
||||
@@ -122,11 +130,34 @@
|
||||
" x-show="home === 2" x-transition>
|
||||
</div>
|
||||
<div class="input-container">
|
||||
<div class="input-performance">
|
||||
<span class="input-performance-point">
|
||||
<p class="monospace" x-text="(time_till_first / 1000).toFixed(2)"></p>
|
||||
<p class="megrim-regular">SEC TO FIRST TOKEN</p>
|
||||
</span>
|
||||
<span class="input-performance-point">
|
||||
<p class="monospace" x-text="tokens_per_second.toFixed(1)"></p>
|
||||
<p class="megrim-regular">TOKENS/SEC</p>
|
||||
</span>
|
||||
<span class="input-performance-point">
|
||||
<p class="monospace" x-text="total_tokens"></p>
|
||||
<p class="megrim-regular">TOKENS</p>
|
||||
</span>
|
||||
</div>
|
||||
<div class="input">
|
||||
<textarea x-ref="inputForm" id="input-form" class="input-form" autofocus rows=1 x-autosize
|
||||
:placeholder="generating ? 'Generating...' : 'Say something'" :disabled="generating" @input="
|
||||
home = (home === 0) ? 1 : home
|
||||
if (cstate.messages.length === 0 && $el.value === '') home = -1;
|
||||
|
||||
if ($el.value !== '') {
|
||||
const messages = [...cstate.messages];
|
||||
messages.push({ role: 'user', content: $el.value });
|
||||
updateTotalTokens(messages);
|
||||
} else {
|
||||
if (cstate.messages.length === 0) total_tokens = 0;
|
||||
else updateTotalTokens(cstate.messages);
|
||||
}
|
||||
" x-effect="
|
||||
console.log(generating);
|
||||
if (!generating) $nextTick(() => {
|
||||
|
||||
@@ -13,6 +13,11 @@ document.addEventListener("alpine:init", () => {
|
||||
generating: false,
|
||||
endpoint: `${window.location.origin}/v1`,
|
||||
|
||||
// performance tracking
|
||||
time_till_first: 0,
|
||||
tokens_per_second: 0,
|
||||
total_tokens: 0,
|
||||
|
||||
removeHistory(cstate) {
|
||||
const index = this.histories.findIndex((state) => {
|
||||
return state.time === cstate.time;
|
||||
@@ -43,18 +48,37 @@ document.addEventListener("alpine:init", () => {
|
||||
el.style.height = "auto";
|
||||
el.style.height = el.scrollHeight + "px";
|
||||
|
||||
// reset performance tracking
|
||||
const prefill_start = Date.now();
|
||||
let start_time = 0;
|
||||
let tokens = 0;
|
||||
this.tokens_per_second = 0;
|
||||
|
||||
// start receiving server sent events
|
||||
let gottenFirstChunk = false;
|
||||
for await (
|
||||
const chunk of this.openaiChatCompletion(this.cstate.messages)
|
||||
) {
|
||||
if (!gottenFirstChunk) {
|
||||
this.cstate.messages.push({ role: "ai", content: "" });
|
||||
this.cstate.messages.push({ role: "assistant", content: "" });
|
||||
gottenFirstChunk = true;
|
||||
}
|
||||
|
||||
// add chunk to the last message
|
||||
this.cstate.messages[this.cstate.messages.length - 1].content += chunk;
|
||||
|
||||
// calculate performance tracking
|
||||
tokens += 1;
|
||||
this.total_tokens += 1;
|
||||
if (start_time === 0) {
|
||||
start_time = Date.now();
|
||||
this.time_till_first = start_time - prefill_start;
|
||||
} else {
|
||||
const diff = Date.now() - start_time
|
||||
if (diff > 0) {
|
||||
this.tokens_per_second = tokens / (diff / 1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// update the state in histories or add it if it doesn't exist
|
||||
@@ -82,6 +106,16 @@ document.addEventListener("alpine:init", () => {
|
||||
}
|
||||
},
|
||||
|
||||
updateTotalTokens(messages) {
|
||||
fetch(`${this.endpoint}/chat/token/encode`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ messages })
|
||||
}).then(response => response.json()).then(data => {
|
||||
this.total_tokens = data.length;
|
||||
}).catch(console.error);
|
||||
},
|
||||
|
||||
async *openaiChatCompletion(messages) {
|
||||
// stream response
|
||||
const response = await fetch(`${this.endpoint}/chat/completions`, {
|
||||
|
||||
Reference in New Issue
Block a user