github/tinygrad
1
1
Fork 0
mirror of https://github.com/tinygrad/tinygrad.git synced 2026-04-29 03:00:14 -04:00
Code Issues Packages Projects Releases Wiki Activity

oops, forgot some stars

Browse Source
This commit is contained in:
George Hotz
2021-11-30 00:46:14 -05:00
parent 908db3bdea
commit c39824bc62
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
models/transformer.py
View File

@@ -45,9 +45,9 @@ class TransformerBlock:
x = x + x.layernorm().linear(*self.ln2).linear(*self.ff1).gelu().linear(*self.ff2).dropout(0.1)
else:
x = x + self.attn(x).dropout(0.1)
x = x.layernorm().linear(self.ln1)
x = x + x.linear(self.ff1).relu().linear(self.ff2).dropout(0.1)
x = x.layernorm().linear(self.ln2)
x = x.layernorm().linear(*self.ln1)
x = x + x.linear(*self.ff1).relu().linear(*self.ff2).dropout(0.1)
x = x.layernorm().linear(*self.ln2)
return x
class Transformer: