From 3cc6ae0d8578eb12cfeec1ccf092b41ad480aa40 Mon Sep 17 00:00:00 2001
From: chenyu <chenyu@fastmail.com>
Date: Mon, 3 Jun 2024 09:49:59 -0400
Subject: [PATCH] layernorm backward is indepedent of its mean (#4806)

---
 tinygrad/tensor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py
index 8a59bc1efd..96cea73c02 100644
--- a/tinygrad/tensor.py
+++ b/tinygrad/tensor.py
@@ -2590,7 +2590,7 @@ class Tensor:
     print(t.mean().item(), t.std().item())
     ```
     """
-    y = (self - self.mean(axis, keepdim=True))
+    y = (self - self.detach().mean(axis, keepdim=True))
     return y.mul((y*y).mean(axis, keepdim=True).add(eps).rsqrt())
 
   def batchnorm(self, weight:Optional[Tensor], bias:Optional[Tensor], mean:Tensor, invstd:Tensor, axis:Union[int,Tuple[int,...]]=1) -> Tensor: