Support for slerp merging models (#455)

* support for slerp merging models

* docs

* update docs

* format'
This commit is contained in:
Awni Hannun
2024-02-19 20:37:15 -08:00
committed by GitHub
parent 8c9148a8fd
commit 8fd953ee2b
16 changed files with 329 additions and 15 deletions

View File

@@ -11,7 +11,6 @@ from .base import BaseModelArgs
@dataclass
class ModelArgs(BaseModelArgs):
model_type: str
vocab_size: int
vocab_size: int = 32000
max_position_embeddings: int = 4096 * 32
hidden_size: int = 4096
@@ -260,3 +259,7 @@ class Model(nn.Module):
):
out, cache = self.model(inputs, cache)
return self.lm_head(out), cache
@property
def layers(self):
return self.model.layers