Upload folder using huggingface_hub

README.md

@@ -37,6 +37,10 @@ save_folder = "/tmp/yujiepan/baguettotron-tiny-random"
 tokenizer = AutoTokenizer.from_pretrained(
     source_model_id, trust_remote_code=True,
 )
+tokenizer.chat_template = "{% for m in messages %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n<think>\n{% endif %}"
+tokenizer.eos_token = "<|im_end|>"
+tokenizer.bos_token = "<|im_start|>"
+tokenizer.stop_token = "<|im_end|>"
 tokenizer.save_pretrained(save_folder)
 
 config = AutoConfig.from_pretrained(

chat_template.jinja

@@ -0,0 +1,5 @@
+{% for m in messages %}<|im_start|>{{ m['role'] }}
+{{ m['content'] }}<|im_end|>
+{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+<think>
+{% endif %}

special_tokens_map.json

@@ -46,20 +46,8 @@
     "⟨H≈1.7⟩",
     "⟨H≈1.8⟩"
   ],
-  "bos_token": {
-    "content": "<|begin_of_text|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|end_of_text|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
   "pad_token": {
     "content": "[PAD]",
     "lstrip": false,

tokenizer_config.json

@@ -440,9 +440,9 @@
     "⟨H≈1.7⟩",
     "⟨H≈1.8⟩"
   ],
-  "bos_token": "<|begin_of_text|>",
+  "bos_token": "<|im_start|>",
   "clean_up_tokenization_spaces": true,
-  "eos_token": "<|end_of_text|>",
+  "eos_token": "<|im_end|>",
   "extra_special_tokens": {},
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "[PAD]",