From 64e53e84158e5c2e712de10566395c5b8a3edf5f Mon Sep 17 00:00:00 2001 From: Juarez Bochi Date: Mon, 18 Dec 2023 15:05:05 -0500 Subject: [PATCH] Pass ln2 to cross attention --- t5/t5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t5/t5.py b/t5/t5.py index a166d4c4..8f344d11 100644 --- a/t5/t5.py +++ b/t5/t5.py @@ -210,7 +210,7 @@ class TransformerDecoderLayer(nn.Module): x = x + y y = self.ln2(x) - y, _ = self.cross_attention(x, memory, memory, memory_mask) + y, _ = self.cross_attention(y, memory, memory, memory_mask) x = x + y y = self.ln3(x)