From 7f8c961287442ec1b78bedbba8e34251ec6defa2 Mon Sep 17 00:00:00 2001
From: Angelos Katharopoulos <a_katharopoulos@apple.com>
Date: Wed, 28 Aug 2024 14:47:33 -0700
Subject: [PATCH] Fix setattr for the TokenizerWrapper (#961)

---
 llms/mlx_lm/tokenizer_utils.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/llms/mlx_lm/tokenizer_utils.py b/llms/mlx_lm/tokenizer_utils.py
index 6caad629..04bbbcc5 100644
--- a/llms/mlx_lm/tokenizer_utils.py
+++ b/llms/mlx_lm/tokenizer_utils.py
@@ -252,9 +252,19 @@ class TokenizerWrapper:
     def __getattr__(self, attr):
         if attr == "detokenizer":
             return self._detokenizer
+        elif attr.startswith("_"):
+            return self.__getattribute__(attr)
         else:
             return getattr(self._tokenizer, attr)
 
+    def __setattr__(self, attr, value):
+        if attr == "detokenizer":
+            raise AttributeError("Cannot set the detokenizer.")
+        elif attr.startswith("_"):
+            super().__setattr__(attr, value)
+        else:
+            setattr(self._tokenizer, attr, value)
+
 
 def _match(a, b):
     if type(a) != type(b):