From b763ad3829384e0080c4c17e24402fc8f39033f8 Mon Sep 17 00:00:00 2001 From: Anchen Date: Mon, 1 Jan 2024 02:17:36 +1100 Subject: [PATCH] remove unused num_key_value_heads --- llms/yayi2/yayi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/llms/yayi2/yayi.py b/llms/yayi2/yayi.py index 62dadae8..29dfd4cf 100644 --- a/llms/yayi2/yayi.py +++ b/llms/yayi2/yayi.py @@ -41,7 +41,6 @@ class Attention(nn.Module): def __init__(self, args: ModelArgs): super().__init__() self.num_attention_heads: int = args.num_attention_heads - self.num_key_value_heads: int = args.num_key_value_heads self.hidden_size = args.hidden_size self.head_dim = args.hidden_size // args.num_attention_heads