refactor(qwen): moving qwen into mlx-lm (#312)

* refactor(qwen): moving qwen into mlx-lm

* chore: update doc

* chore: fix type hint

* add qwen model support in convert

* chore: fix doc

* chore: only load model in quantize_model

* chore: make the convert script only copy tokenizer files instead of load it and save

* chore: update docstring

* chore: remove unnecessary try catch

* chore: clean up for tokenizer and update  transformers 4.37

* nits in README

---------

Co-authored-by: Awni Hannun <awni@apple.com>

llms/mlx_lm/generate.py

@@ -21,6 +21,17 @@ def setup_arg_parser():
         default="mlx_model",
         help="The path to the local model directory or Hugging Face repo.",
     )
+    parser.add_argument(
+        "--trust-remote-code",
+        action="store_true",
+        help="Enable trusting remote code for tokenizer",
+    )
+    parser.add_argument(
+        "--eos-token",
+        type=str,
+        default=None,
+        help="End of sequence token for tokenizer",
+    )
     parser.add_argument(
         "--prompt", default=DEFAULT_PROMPT, help="Message to be processed by the model"
     )
@@ -40,7 +51,13 @@ def setup_arg_parser():
 
 def main(args):
     mx.random.seed(args.seed)
-    model, tokenizer = load(args.model)
+
+    # Building tokenizer_config
+    tokenizer_config = {"trust_remote_code": True if args.trust_remote_code else None}
+    if args.eos_token is not None:
+        tokenizer_config["eos_token"] = args.eos_token
+
+    model, tokenizer = load(args.model, tokenizer_config=tokenizer_config)
     print("=" * 10)
     print("Prompt:", args.prompt)
     prompt = tokenizer.encode(args.prompt)